Feature/maintenance (#916)

* Maintenance mode
* fix(hls): release origin slot before next origin I/O
This commit is contained in:
euzu
2026-10-13 22:30:21 +02:00
committed by GitHub
parent 588b954eeb
commit 69040f022b
122 changed files with 3313 additions and 147 deletions
+33 -1
View File
@@ -11,7 +11,7 @@ use std::{
path::{Path, PathBuf},
};
use tuliprox_core::{
model::{ConfigInput, EpgSource, EpgSourceType, PersistedEpgSource, PersistedEpgSourceKind},
model::{ConfigInput, EpgConfig, EpgSource, EpgSourceType, PersistedEpgSource, PersistedEpgSourceKind},
utils::{add_prefix_to_filename, prepare_file_path, request},
};
use tuliprox_repository::get_input_storage_path;
@@ -191,6 +191,9 @@ pub async fn get_xmltv<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
) -> (Option<TVGuide>, Vec<TuliproxError>) {
match &input.epg {
None => (None, vec![]),
Some(epg_config) if ctx.is_input_in_maintenance(&input.name) => {
persisted_epg_for_maintenance(ctx, input, epg_config, storage_dir).await
}
Some(epg_config) => {
let mut errors = vec![];
let mut file_paths = vec![];
@@ -226,6 +229,35 @@ pub async fn get_xmltv<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
}
}
/// The EPG files fetched last, without contacting any source and without cleanup:
/// deleting "unlisted" files here would throw away exactly what maintenance keeps.
async fn persisted_epg_for_maintenance<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
ctx: &PlaylistProcessingContext<E, M>,
input: &ConfigInput,
epg_config: &EpgConfig,
storage_dir: &str,
) -> (Option<TVGuide>, Vec<TuliproxError>) {
debug!("Input '{}' is in maintenance; using its persisted epg", input.name);
let mut errors = vec![];
let mut file_paths = vec![];
for epg_source in &epg_config.sources {
let Ok(path) = get_input_raw_epg_file_path(epg_source, input, storage_dir).await else {
continue;
};
if tokio::fs::metadata(&path).await.is_ok_and(|metadata| metadata.is_file()) {
match persisted_source_from_config(epg_source, path) {
Ok(persisted) => file_paths.push(persisted),
Err(err) => errors.push(err),
}
}
}
if file_paths.is_empty() {
(None, errors)
} else {
(Some(TVGuide::new(file_paths).with_file_locks(std::sync::Arc::clone(&ctx.config.file_locks))), errors)
}
}
fn persisted_source_from_config(
epg_source: &EpgSource,
file_path: PathBuf,
@@ -114,6 +114,8 @@ pub struct PlaylistProcessingContext<E: EventSink, M: MetadataUpdateSink = NoopM
/// Optional request-local behavior for one manually selected input.
pub input_refresh: Option<InputRefreshOverride>,
pub(crate) library_update_mode: LibraryUpdateMode,
/// Inputs in maintenance: their persisted state is reused, their provider is not contacted.
pub maintenance: Option<Arc<dyn tuliprox_core::model::MaintenanceGate>>,
}
#[cfg(test)]
@@ -306,6 +306,12 @@ pub(crate) async fn download_input<E: EventSink + Clone + 'static, M: MetadataUp
if already_processed {
// Use empty results, will load from disk below
in_run_reuse_playlist_download_result()
} else if ctx.is_input_in_maintenance(&input.name) {
// Keep serving what was fetched last; the provider is out of service. This also
// covers the Stalker refresh, which only runs inside the provider download.
debug!("Input '{}' is in maintenance; using its persisted playlist", input.name);
mark_as_processed = true;
cached_playlist_download_result(input, refresh_policy)
} else if ctx.pre_processed_inputs.as_ref().is_some_and(|s| s.contains(&input.name)) {
// Input was already processed in a prior session; skip download and load from disk.
// Mark only after load succeeds (or fails) to avoid exposing a half-ready state.
@@ -335,7 +341,12 @@ pub(crate) async fn download_input<E: EventSink + Clone + 'static, M: MetadataUp
// Defensive fallback: if cache metadata says "valid" but persisted data is unreadable,
// retry once before forcing a refresh.
let must_force_refresh = cached_error.is_some();
if must_force_refresh {
if must_force_refresh && ctx.is_input_in_maintenance(&input.name) {
// A refresh would contact the provider; without a persisted playlist the input
// contributes nothing until maintenance ends.
warn!("Input '{}' is in maintenance and has no readable persisted playlist; skipping it", input.name);
preloaded_playlist = Some((MemoryPlaylistSource::default().into_source(), None));
} else if must_force_refresh {
warn!("Input '{}' cache hit produced unreadable playlist; retrying cached load once", input.name);
let (retry_playlist, retry_error) = load_cached_input_playlist(ctx, input).await;
if retry_error.is_none() {
@@ -505,6 +516,12 @@ async fn download_m3u_alias_playlists<E: EventSink + Clone + 'static, M: Metadat
input: &ConfigInput,
) -> Vec<TuliproxError> {
let Some(aliases) = input.get_enabled_aliases() else { return vec![] };
// Maintenance is keyed by the parent input; an alias carries its own name and would
// otherwise contact the provider. Its persisted playlist stays in place.
if ctx.is_input_in_maintenance(&input.name) {
debug!("Input '{}' is in maintenance; skipping its M3U alias downloads", input.name);
return vec![];
}
let mut errors = Vec::new();
for alias in aliases {
@@ -466,11 +466,16 @@ impl<E: EventSink + Clone, M: MetadataUpdateSink> Clone for PlaylistProcessingCo
had_quality_rejections: Arc::clone(&self.had_quality_rejections),
input_refresh: self.input_refresh,
library_update_mode: self.library_update_mode,
maintenance: self.maintenance.clone(),
}
}
}
impl<E: EventSink + Clone + 'static, M: MetadataUpdateSink> PlaylistProcessingContext<E, M> {
pub fn is_input_in_maintenance(&self, input_name: &str) -> bool {
self.maintenance.as_ref().is_some_and(|gate| gate.is_in_maintenance(input_name))
}
#[must_use]
pub fn refresh_policy(&self, input_id: u16) -> InputRefreshPolicy {
self.input_refresh
@@ -146,6 +146,7 @@ pub struct ProcessingRun<
acquired_permit: Option<tuliprox_core::model::UpdateGuardPermit>,
input_refresh: Option<InputRefreshOverride>,
library_update_mode: LibraryUpdateMode,
maintenance: Option<Arc<dyn tuliprox_core::model::MaintenanceGate>>,
}
impl<E: EventSink + Clone + 'static> ProcessingRun<E, NoBootstrap, NoopMetadataSink> {
@@ -178,6 +179,7 @@ impl<E: EventSink + Clone + 'static> ProcessingRun<E, NoBootstrap, NoopMetadataS
acquired_permit: None,
input_refresh: None,
library_update_mode: LibraryUpdateMode::ExistingCatalog,
maintenance: None,
}
}
}
@@ -205,6 +207,7 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
acquired_permit: self.acquired_permit,
input_refresh: self.input_refresh,
library_update_mode: self.library_update_mode,
maintenance: self.maintenance,
}
}
@@ -255,6 +258,7 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
acquired_permit: self.acquired_permit,
input_refresh: self.input_refresh,
library_update_mode: self.library_update_mode,
maintenance: self.maintenance,
}
}
@@ -291,6 +295,13 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
self
}
/// Inputs taken out of service; their provider is not contacted during this run.
#[must_use]
pub fn with_maintenance<G: tuliprox_core::model::MaintenanceGate + 'static>(mut self, gate: Arc<G>) -> Self {
self.maintenance = Some(gate);
self
}
/// Applies one manual policy override without changing the input configuration.
#[must_use]
pub fn with_input_refresh(mut self, input_refresh: impl Into<Option<InputRefreshOverride>>) -> Self {
@@ -319,6 +330,7 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
acquired_permit,
input_refresh,
library_update_mode,
maintenance,
} = run;
let max_update_duration = Duration::from_secs(PLAYLIST_UPDATE_MAX_DURATION_SECS);
@@ -396,6 +408,7 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)),
input_refresh,
library_update_mode,
maintenance,
};
let start_time = Instant::now();
@@ -609,6 +609,10 @@ pub(crate) async fn playlist_resolve<E: EventSink + Clone + 'static, M: Metadata
provider_fpl: &mut FetchedPlaylist<'_>,
processed_fpl: &mut FetchedPlaylist<'_>,
) {
// Resolve and probe talk to the provider; an input in maintenance keeps what it has.
if ctx.is_input_in_maintenance(&processed_fpl.input.name) {
return;
}
playlist_resolve_series(ctx, target, errors, pipe, provider_fpl, processed_fpl).await;
playlist_resolve_vod(ctx, target, errors, provider_fpl, processed_fpl).await;
playlist_probe(ctx, target, processed_fpl).await;
@@ -73,6 +73,7 @@ pub(super) fn processing_context(
stalker_refresh_mode: StalkerRefreshMode::Complete,
partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)),
had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)),
maintenance: None,
input_refresh: None,
library_update_mode: LibraryUpdateMode::ExistingCatalog,
}
@@ -97,6 +97,7 @@ fn processing_context() -> PlaylistProcessingContext<shared::model::NoopSink> {
stalker_refresh_mode: StalkerRefreshMode::Complete,
partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)),
had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)),
maintenance: None,
}
}
@@ -240,6 +240,7 @@ fn processing_context_with_events<E: EventSink>(storage_dir: &Path, events: E) -
stalker_refresh_mode: StalkerRefreshMode::Complete,
partial_refresh: Arc::new(AtomicBool::new(false)),
had_quality_rejections: Arc::new(AtomicBool::new(false)),
maintenance: None,
}
}
@@ -1617,3 +1618,104 @@ mod m3u_update_quality;
mod manual_update_integration;
mod persisted_input_status;
mod staged_completion;
struct FixedGate(&'static str);
impl tuliprox_core::model::MaintenanceGate for FixedGate {
fn is_in_maintenance(&self, input_name: &str) -> bool { input_name == self.0 }
}
fn quality_95() -> ConfigInputUpdateQualityDto { ConfigInputUpdateQualityDto { live: 95, vod: 95, series: 95 } }
#[tokio::test]
async fn maintenance_input_uses_persisted_playlist_without_download() {
let temp = tempfile::tempdir().expect("temporary storage");
let server = TestXtreamServer::start([2, 2, 2]);
let mut ctx = processing_context(temp.path());
// cache_duration_seconds = 0: without maintenance this input is downloaded.
let input = test_input(&server.base_url, quality_95(), 0, &[]);
seed_baseline(&ctx, &input).await;
ctx.maintenance = Some(Arc::new(FixedGate("quality-provider")));
let mut result = download_input(&ctx, &input, false).await;
assert!(result.errors.is_empty(), "maintenance must not fail the input: {:?}", result.errors);
let baseline_channels: usize = baseline_groups().iter().map(|group| group.channels.len()).sum();
assert_eq!(result.source.get_channel_count(), baseline_channels, "the persisted playlist is served");
assert_requested_actions(server.finish(), &[]);
}
#[tokio::test]
async fn maintenance_input_without_cache_contributes_nothing() {
let temp = tempfile::tempdir().expect("temporary storage");
let server = TestXtreamServer::start([2, 2, 2]);
let mut ctx = processing_context(temp.path());
let input = test_input(&server.base_url, quality_95(), 0, &[]);
ctx.maintenance = Some(Arc::new(FixedGate("quality-provider")));
let mut result = download_input(&ctx, &input, false).await;
assert!(result.errors.is_empty(), "a missing cache is not an error in maintenance: {:?}", result.errors);
assert_eq!(result.source.get_channel_count(), 0);
assert_requested_actions(server.finish(), &[]);
}
#[tokio::test]
async fn input_not_in_maintenance_still_downloads() {
let temp = tempfile::tempdir().expect("temporary storage");
let server = TestXtreamServer::start([2, 2, 2]);
let mut ctx = processing_context(temp.path());
let input = test_input(&server.base_url, quality_95(), 0, &[]);
seed_baseline(&ctx, &input).await;
ctx.maintenance = Some(Arc::new(FixedGate("some-other-input")));
let mut result = download_input(&ctx, &input, false).await;
assert!(result.errors.is_empty(), "{:?}", result.errors);
assert_eq!(result.source.get_channel_count(), 6);
assert!(!server.finish().is_empty(), "the provider was contacted");
}
#[tokio::test]
async fn maintenance_input_skips_epg_download_and_keeps_existing_file() {
use tuliprox_core::model::{EpgConfig, EpgSource, EpgSourceType};
let temp = tempfile::tempdir().expect("temporary storage");
let server = TestXtreamServer::start([2, 2, 2]);
let mut ctx = processing_context(temp.path());
let mut input = (*test_input(&server.base_url, quality_95(), 0, &[])).clone();
let epg_source = EpgSource {
source_type: EpgSourceType::Xmltv,
url: format!("{}/xmltv.php", server.base_url),
priority: 0,
logo_override: false,
channel_id: None,
channel_title: None,
match_names: Vec::new(),
ics: None,
};
input.epg = Some(EpgConfig { sources: vec![epg_source.clone()], smart_match: None });
let storage_dir = temp.path().to_string_lossy().into_owned();
let epg_path =
crate::epg::get_input_raw_epg_file_path(&epg_source, &input, &storage_dir).await.expect("epg file path");
tokio::fs::write(&epg_path, "<tv></tv>").await.expect("seed persisted epg");
ctx.maintenance = Some(Arc::new(FixedGate("quality-provider")));
let (guide, errors) = crate::epg::get_xmltv(&ctx, &input, None, &storage_dir).await;
assert!(errors.is_empty(), "{errors:?}");
assert!(guide.is_some(), "the persisted epg is used");
assert!(epg_path.exists(), "maintenance must not clean up the persisted epg");
assert_requested_actions(server.finish(), &[]);
}
#[test]
fn resolve_stage_checks_maintenance_before_any_provider_work() {
let source = include_str!("../target.rs");
let start = source.find("pub(crate) async fn playlist_resolve<").expect("playlist_resolve");
let body = &source[start..];
let body = &body[..body.find("\n}\n").expect("function end")];
let gate = body.find("if ctx.is_input_in_maintenance(").expect("maintenance gate in playlist_resolve");
let series = body.find("playlist_resolve_series(").expect("series resolve call");
assert!(gate < series, "the gate must run before resolve and probe");
}