feat(update): add per-cluster quality safeguards (#855)

* feat(update): add per-input actions and pipeline transparency

Squash feature/cluster-update-quality onto upstream/develop.

Include request-local update policies, persisted input and cluster status, library rescan and target rebuild integration, and the update-card view. Align the M3U request regression with the existing Force capability.

* del(build): temporary dev changes

* fix(update): address playlist review findings

* fix(update): rename Shows to Series

* Playlist update refactor

* Playlist update refactor

---------

Co-authored-by: euzu <euzu@proton.me>
This commit is contained in:
knylbyte
2026-09-14 16:44:23 +02:00
committed by GitHub
co-authored by euzu
parent a25801e906
commit 33ed942c16
117 changed files with 27195 additions and 1340 deletions
+14 -6
View File
@@ -66,16 +66,19 @@ pub async fn download_library_playlist(
input: &ConfigInput,
) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let config = &*app_config.config.load();
let Some(library_config) = config.library.as_ref() else { return (vec![], vec![]) };
if !library_config.enabled {
return (vec![], vec![]);
}
let failure = |error: String| (vec![], vec![TuliproxError::RepositoryLibrary(error)]);
let Some(library_config) = config.library.as_ref().filter(|library| library.enabled) else {
return failure("Library is disabled or unavailable".into());
};
let api_base_path =
concat_path_leading_slash(config.web_ui.as_ref().and_then(|w| w.path.as_deref()).unwrap_or(""), "api/v1");
let storage_path =
resolve_metadata_storage_path(config.metadata_update.as_ref(), &config.storage_dir).join("library");
let mut metadata_iter = MetadataAsyncIter::new(&storage_path).await;
let mut metadata_iter = match MetadataAsyncIter::try_new(&storage_path).await {
Ok(iter) => iter,
Err(err) => return failure(format!("Failed to open Library catalog: {err}")),
};
let mut group_movies = PlaylistGroup {
id: 0,
title: library_config.playlist.movie_category.clone(),
@@ -88,7 +91,12 @@ pub async fn download_library_playlist(
channels: vec![],
xtream_cluster: XtreamCluster::Series,
};
while let Some(entry) = metadata_iter.next().await {
loop {
let entry = match metadata_iter.try_next().await {
Ok(Some(entry)) => entry,
Ok(None) => break,
Err(err) => return failure(format!("Failed to read Library catalog: {err}")),
};
match entry.metadata {
MediaMetadata::Movie(_) => {
to_playlist_item(
@@ -0,0 +1,286 @@
use crate::input_cache::{self, ClusterState, InputStatus};
use log::warn;
use shared::model::{
EventMessage, EventSink, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, XtreamCluster,
};
use tuliprox_core::model::{ClusterForceUpdate, ClusterUpdateRejection, ConfigInput};
use tuliprox_iptv::provider::PlaylistFetch;
/// Cache entries whose state is determined by one completed provider fetch.
#[derive(Clone, Copy)]
pub(super) enum CacheStatusScope<'a> {
Default,
RequestedClusters(&'a [XtreamCluster]),
}
/// Applies the observable and cache-state effects of a completed provider fetch.
///
/// Quality rejections remain separate from technical errors. For per-cluster
/// fetches, only requested clusters are updated, so valid cached clusters keep
/// both their state and timestamp.
pub(super) fn apply_playlist_fetch_outcome<E: EventSink>(
events: &E,
run_id: &PlaylistUpdateRunId,
execution_order: PlaylistUpdateRunOrder,
input: &ConfigInput,
status: &mut InputStatus,
cache_scope: CacheStatusScope<'_>,
fetch: &PlaylistFetch,
) -> bool {
report_quality_rejections(events, run_id, execution_order, input, &fetch.quality_rejections);
if fetch.persisted {
report_force_updates(events, run_id, execution_order, input, &fetch.force_updates);
}
let technical_failure = fetch.partial || !fetch.errors.is_empty();
match cache_scope {
CacheStatusScope::RequestedClusters(requested_clusters) => {
for cluster in requested_clusters {
let rejected = fetch.quality_rejections.iter().any(|rejection| rejection.cluster == *cluster);
let state = if technical_failure || rejected { ClusterState::Failed } else { ClusterState::Ok };
input_cache::update_cluster_status(status, cluster.as_ref(), state);
}
!requested_clusters.is_empty()
}
CacheStatusScope::Default => {
let default_state = if technical_failure || !fetch.quality_rejections.is_empty() {
ClusterState::Failed
} else {
ClusterState::Ok
};
input_cache::update_cluster_status(status, "default", default_state);
// M3U still acquires/caches one document, but its completed decisions
// and reload snapshots are per cluster. A rejected cluster invalidates
// the document cache without labelling accepted siblings Failed.
if input.get_download_input_type() == shared::model::InputType::M3u {
for cluster in super::ingest::PIPELINE_TRANSPARENCY_CLUSTERS {
if super::ingest::cluster_is_configured(input, cluster) {
let rejected = fetch.quality_rejections.iter().any(|item| item.cluster == cluster);
let state = if technical_failure || rejected { ClusterState::Failed } else { ClusterState::Ok };
input_cache::update_cluster_status(status, cluster.as_ref(), state);
}
}
}
for rejection in &fetch.quality_rejections {
input_cache::update_cluster_status(status, rejection.cluster.as_ref(), ClusterState::Failed);
}
true
}
}
}
pub(super) fn report_force_updates<E: EventSink>(
events: &E,
run_id: &PlaylistUpdateRunId,
execution_order: PlaylistUpdateRunOrder,
input: &ConfigInput,
force_updates: &[ClusterForceUpdate],
) {
for force_update in force_updates {
let message = format!(
"Input '{}' cluster '{}' force-published: candidate={} configured_threshold={}",
input.name,
quality_cluster_name(force_update.cluster),
force_update.candidate_count,
force_update.configured_threshold
);
log::info!("{message}");
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input(
run_id.clone(),
execution_order,
input.id,
input.name.to_string(),
message,
)));
}
}
fn report_quality_rejections<E: EventSink>(
events: &E,
run_id: &PlaylistUpdateRunId,
execution_order: PlaylistUpdateRunOrder,
input: &ConfigInput,
rejections: &[ClusterUpdateRejection],
) {
for rejection in rejections {
let message = format!(
"Input '{}' cluster '{}' rejected: current={} candidate={} threshold={} quality={}; retaining previous cluster",
input.name,
quality_cluster_name(rejection.cluster),
rejection.current_count,
rejection.candidate_count,
rejection.threshold,
rejection.quality
);
warn!("{message}");
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input(
run_id.clone(),
execution_order,
input.id,
input.name.to_string(),
message,
)));
}
}
const fn quality_cluster_name(cluster: XtreamCluster) -> &'static str {
match cluster {
XtreamCluster::Live => "live",
XtreamCluster::Video => "vod",
XtreamCluster::Series => "series",
}
}
#[cfg(test)]
mod tests {
use super::{apply_playlist_fetch_outcome, CacheStatusScope};
use crate::input_cache::{is_cache_valid, ClusterState, ClusterStatus, InputStatus};
use shared::model::{
EventMessage, EventSink, InputType, PlaylistUpdateRunId, PlaylistUpdateRunOrder, XtreamCluster,
};
use std::sync::{Arc, Mutex};
use tuliprox_core::model::{ClusterForceUpdate, ClusterUpdateRejection, ConfigInput};
use tuliprox_iptv::provider::PlaylistFetch;
#[derive(Clone, Default)]
struct CollectSink(Arc<Mutex<Vec<EventMessage>>>);
impl EventSink for CollectSink {
fn emit(&self, event: EventMessage) { self.0.lock().expect("event sink lock").push(event); }
}
fn rejection(cluster: XtreamCluster) -> ClusterUpdateRejection {
ClusterUpdateRejection { cluster, current_count: 12_543, candidate_count: 217, threshold: 90, quality: 1 }
}
fn run_id() -> PlaylistUpdateRunId { "fetch-outcome-run".into() }
fn execution_order() -> PlaylistUpdateRunOrder { PlaylistUpdateRunOrder::from(41) }
#[test]
fn playlist_update_run_mixed_cluster_fetch_reports_correlated_rejection() {
let input = ConfigInput { name: Arc::from("provider-a"), input_type: InputType::Xtream, ..Default::default() };
let events = CollectSink::default();
let mut status = InputStatus::default();
status
.clusters
.insert("series".to_string(), ClusterStatus { status: ClusterState::Ok, timestamp: 17, last_update: None });
let fetch = PlaylistFetch::groups(Vec::new()).with_quality_rejections(vec![rejection(XtreamCluster::Video)]);
let changed = apply_playlist_fetch_outcome(
&events,
&run_id(),
execution_order(),
&input,
&mut status,
CacheStatusScope::RequestedClusters(&[XtreamCluster::Live, XtreamCluster::Video]),
&fetch,
);
assert!(changed);
assert_eq!(
status.clusters.get(XtreamCluster::Live.as_ref()).map(|entry| &entry.status),
Some(&ClusterState::Ok)
);
assert_eq!(
status.clusters.get(XtreamCluster::Video.as_ref()).map(|entry| &entry.status),
Some(&ClusterState::Failed)
);
let series = status.clusters.get("series").expect("cached series status");
assert_eq!(series.status, ClusterState::Ok);
assert_eq!(series.timestamp, 17);
assert!(!status.clusters.contains_key("default"));
let emitted = events.0.lock().expect("event sink lock");
assert_eq!(emitted.len(), 1);
let EventMessage::PlaylistUpdateProgress(progress) = &emitted[0] else {
panic!("expected playlist update progress event");
};
assert_eq!(progress.run_id.as_ref().map(AsRef::as_ref), Some("fetch-outcome-run"));
assert_eq!(progress.execution_order, Some(execution_order()));
assert_eq!(progress.target, "provider-a");
assert_eq!(
progress.message,
"Input 'provider-a' cluster 'vod' rejected: current=12543 candidate=217 threshold=90 quality=1; retaining previous cluster"
);
}
#[test]
fn synthetic_default_fetch_invalidates_default_and_cluster_cache_after_rejection() {
let input = ConfigInput { name: Arc::from("provider-a"), input_type: InputType::Stalker, ..Default::default() };
let events = CollectSink::default();
let mut status = InputStatus::default();
status.clusters.insert(
"default".to_string(),
ClusterStatus { status: ClusterState::Ok, timestamp: 23, last_update: None },
);
let fetch = PlaylistFetch::groups(Vec::new()).with_quality_rejections(vec![rejection(XtreamCluster::Series)]);
let changed = apply_playlist_fetch_outcome(
&events,
&run_id(),
execution_order(),
&input,
&mut status,
CacheStatusScope::Default,
&fetch,
);
assert!(changed);
let default = status.clusters.get("default").expect("default cache status");
assert_eq!(default.status, ClusterState::Failed);
assert!(!is_cache_valid(&status, "default", u64::MAX));
assert_eq!(
status.clusters.get(XtreamCluster::Series.as_ref()).map(|entry| &entry.status),
Some(&ClusterState::Failed)
);
}
#[test]
fn playlist_update_run_force_publication_emits_correlated_audit_progress() {
let input = ConfigInput { name: Arc::from("provider-a"), input_type: InputType::Xtream, ..Default::default() };
let events = CollectSink::default();
let mut status = InputStatus::default();
status.clusters.insert(
XtreamCluster::Video.as_ref().to_string(),
ClusterStatus { status: ClusterState::Failed, timestamp: 17, last_update: None },
);
let fetch = PlaylistFetch::default()
.with_force_updates(vec![ClusterForceUpdate {
cluster: XtreamCluster::Video,
candidate_count: 12_450,
configured_threshold: 95,
}])
.persisted(true);
let changed = apply_playlist_fetch_outcome(
&events,
&run_id(),
execution_order(),
&input,
&mut status,
CacheStatusScope::RequestedClusters(&[XtreamCluster::Video]),
&fetch,
);
assert!(changed);
assert_eq!(
status.clusters.get(XtreamCluster::Video.as_ref()).map(|entry| &entry.status),
Some(&ClusterState::Ok)
);
let emitted = events.0.lock().expect("event sink lock");
assert_eq!(emitted.len(), 1);
let EventMessage::PlaylistUpdateProgress(progress) = &emitted[0] else {
panic!("expected playlist update progress event");
};
assert_eq!(progress.run_id.as_ref().map(AsRef::as_ref), Some("fetch-outcome-run"));
assert_eq!(progress.execution_order, Some(execution_order()));
assert_eq!(progress.target, "provider-a");
assert_eq!(
progress.message,
"Input 'provider-a' cluster 'vod' force-published: candidate=12450 configured_threshold=95"
);
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,75 @@
use super::{InputDownloadResult, InputJobResult, MetadataUpdateSink, PlaylistProcessingContext};
use crate::input_cache::{load_input_status, resolve_input_storage_path, save_input_status};
use shared::model::{EventSink, PersistedPlaylistUpdateInputResult, PlaylistUpdateState};
use std::{
sync::Arc,
time::{SystemTime, UNIX_EPOCH},
};
pub(super) async fn persist_input_job_result<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
ctx: &PlaylistProcessingContext<E, M>,
result: &InputJobResult,
) {
// Successful in-run reuse has no new acquisition facts and must not replace the
// original result (in particular Partial) or advance its completion timestamp.
let state = result.update_state();
if result.input_telemetry.is_some() || state == PlaylistUpdateState::Failure {
persist_input_completion(ctx, result.input_id, &result.input_name, state).await;
}
}
pub(super) async fn complete_staged_input<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
ctx: &PlaylistProcessingContext<E, M>,
input: &tuliprox_core::model::ConfigInput,
result: &mut InputDownloadResult,
) {
let state = result.update_state();
if result.input_telemetry.is_some() || state == PlaylistUpdateState::Failure {
persist_input_completion(ctx, input.id, &input.name, state).await;
// Report the staged input's own facts before its parent consumes them.
// Successful in-run reuse has no new acquisition and stays silent.
super::ingest::report_input_completion(
&ctx.events,
&ctx.run_id,
ctx.execution_order,
input.id,
&input.name,
state,
result.input_telemetry.as_ref(),
);
}
}
pub(super) async fn persist_input_completion<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
ctx: &PlaylistProcessingContext<E, M>,
input_id: u16,
input_name: &Arc<str>,
state: PlaylistUpdateState,
) {
// Same per-input lock and canonical writer as acquisition. Called after its
// own critical section (the staged caller still owns the parent lock).
let _input_lock = ctx.get_input_lock(input_name).await;
let storage_dir = ctx.config.config.load().storage_dir.clone();
let path = resolve_input_storage_path(&storage_dir, input_name).await;
{
// Shared by source contexts of this run only; never compare with the
// previous run's persisted result. Weaker/equal completions do not move
// the timestamp belonging to an already recorded stronger result.
let mut completions = ctx.input_completions.lock().await;
if let Some(current) = completions.get(&input_id) {
use PlaylistUpdateState::{Failure, Partial, Success};
match (current, state) {
(Failure, _) | (Partial, Partial | Success) | (Success, Success) => return,
(Partial, Failure) | (Success, Partial | Failure) => {}
}
}
completions.insert(input_id, state);
}
let mut status = load_input_status(&path);
status.last_input_update = Some(PersistedPlaylistUpdateInputResult {
state,
timestamp: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(),
});
// Do not touch the cache's cluster status/timestamp or its proven runtime facts.
save_input_status(&path, &status);
}
@@ -0,0 +1,404 @@
mod persisted_input_status {
use super::*;
use crate::processor::playlist::input_status::{persist_input_completion, persist_input_job_result};
use shared::model::PersistedPlaylistUpdateInputResult;
fn acquired_result(state: PlaylistUpdateState) -> InputJobResult {
let mut result = super::super::playlist_update_run_result(
7,
InputJobState::Ready,
vec![],
state == PlaylistUpdateState::Partial,
);
result.input_telemetry = Some(PlaylistUpdateInputTelemetry {
refresh_policy: InputRefreshPolicy::REFRESH,
source: Some(PlaylistUpdateDataSource::Provider),
clusters: Vec::new(),
});
result
}
#[tokio::test]
async fn playlist_update_status_run_precedence_is_isolated_by_stable_input_id() {
let temp = tempfile::tempdir().unwrap();
let ctx = processing_context(temp.path());
let failed = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false);
persist_input_job_result(&ctx, &failed).await;
let mut independent = acquired_result(PlaylistUpdateState::Success);
independent.input_id = 8;
independent.input_name = "independent".intern();
persist_input_job_result(&ctx, &independent).await;
for (result, state) in [(failed, PlaylistUpdateState::Failure), (independent, PlaylistUpdateState::Success)] {
let path =
input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await;
assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, state);
}
}
#[tokio::test]
async fn m3u_update_quality_playlist_update_status_staged_completion_is_independent_of_parent_failure() {
for (parent_fails, staged_fails) in [(false, false), (true, false), (false, true)] {
let temp = tempfile::tempdir().unwrap();
let events = CollectSink::default();
let ctx = processing_context_with_events(temp.path(), events.clone());
let parent_path = temp.path().join("parent.m3u");
let staged_path = temp.path().join("overlay.m3u");
for (path, fails) in [(&parent_path, parent_fails), (&staged_path, staged_fails)] {
if !fails {
std::fs::write(path, "#EXTM3U\n#EXTINF:-1,Fixture\nhttp://stream.example/fixture\n").unwrap();
}
}
let parent = Arc::new(ConfigInput {
id: 7,
name: "parent".intern(),
input_type: InputType::M3u,
url: parent_path.to_string_lossy().into_owned(),
enabled: true,
..ConfigInput::default()
});
let mut staged = ConfigInput {
id: 8,
name: "staged".intern(),
input_type: InputType::Staged,
url: staged_path.to_string_lossy().into_owned(),
enabled: true,
staged: Some(tuliprox_core::model::ConfigInputStaged {
for_input: Some(parent.name.clone()),
clusters: ClusterFlags::Live,
}),
..ConfigInput::default()
};
staged.resolve_staged_download_type();
let staged = Arc::new(staged);
ctx.config.sources.store(Arc::new(SourcesConfig {
inputs: vec![parent.clone(), staged.clone()],
..SourcesConfig::default()
}));
let result = download_input(&ctx, &parent, false).await;
assert_eq!(!result.errors.is_empty(), parent_fails || staged_fails);
let path = input_storage_path(&ctx, &staged).await;
let persisted = input_cache::load_input_status(&path);
assert_eq!(
persisted.last_input_update.unwrap().state,
if staged_fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success }
);
assert_eq!(persisted.clusters.len(), 4);
assert!(persisted.clusters["default"].last_update.is_none(), "document cache and cluster facts stay separate");
for cluster in XTREAM_CLUSTER {
assert!(persisted.clusters[cluster.as_ref()].last_update.is_some());
}
let completions = staged_completion::input_completions(&events, staged.id);
assert_eq!(completions.len(), 1, "real staged download must report its own completion");
assert_eq!(completions[0].state, persisted.last_input_update.map(|result| result.state));
assert_eq!(completions[0].run_id, Some(ctx.run_id.clone()));
assert_eq!(completions[0].execution_order, Some(ctx.execution_order));
let telemetry = completions[0].input_telemetry.as_ref().unwrap();
assert_eq!(telemetry.source, None);
assert!(telemetry.clusters.iter().all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider)));
assert!(staged_completion::input_completions(&events, parent.id).is_empty());
}
}
#[tokio::test]
async fn playlist_update_status_delayed_source_results_keep_strongest_completion_until_next_run() {
for weaker in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial] {
for failure_first in [false, true] {
let temp = tempfile::tempdir().unwrap();
let events = CollectSink::default();
let ctx = processing_context_with_events(temp.path(), events.clone());
let acquired = acquired_result(weaker);
let failed_reuse = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false);
let (first, delayed) = if failure_first { (failed_reuse, acquired) } else { (acquired, failed_reuse) };
let path =
input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &first.input_name).await;
let mut original = input_cache::InputStatus::default();
original.clusters.insert(
"live".into(),
input_cache::ClusterStatus {
status: input_cache::ClusterState::Failed,
timestamp: 123,
last_update: Some(PersistedPlaylistUpdateClusterSnapshot {
source: Some(PlaylistUpdateDataSource::Cache),
policy: Some(InputRefreshPolicy::FORCE),
quality_guard_threshold: Some(95),
..PersistedPlaylistUpdateClusterSnapshot::default()
}),
},
);
input_cache::save_input_status(&path, &original);
let (release, other_input_finished) = tokio::sync::oneshot::channel();
let delayed_ctx = ctx.clone();
// Source A already holds its shared-input result, but cannot drain
// its sorted result list until another input job has completed.
let delayed_source = async move {
let source_results = vec![delayed];
other_input_finished.await.unwrap();
for result in source_results {
persist_input_job_result(&delayed_ctx, &result).await;
report_input_job_completion(
&delayed_ctx.events,
&delayed_ctx.run_id,
delayed_ctx.execution_order,
&result,
);
}
};
let first_source = async {
persist_input_job_result(&ctx, &first).await;
report_input_job_completion(&ctx.events, &ctx.run_id, ctx.execution_order, &first);
let mut persisted = input_cache::load_input_status(&path);
assert_eq!(persisted.last_input_update.unwrap().state, first.update_state());
// Distinct sentinel proves a delayed weaker result does not
// advance the stronger completion's timestamp, without sleeps.
persisted.last_input_update.as_mut().unwrap().timestamp = 456;
input_cache::save_input_status(&path, &persisted);
release.send(()).unwrap();
};
tokio::join!(delayed_source, first_source);
let final_status = input_cache::load_input_status(&path);
let completion = final_status.last_input_update.unwrap();
assert_eq!(
completion.state,
PlaylistUpdateState::Failure,
"weaker={weaker:?}, failure_first={failure_first}"
);
if failure_first {
assert_eq!(completion.timestamp, 456);
}
assert_eq!(final_status.clusters, original.clusters);
let states: Vec<_> = events
.0
.lock()
.unwrap()
.iter()
.filter_map(|event| match event {
EventMessage::PlaylistUpdateProgress(progress) => progress.state,
_ => None,
})
.collect();
assert_eq!(
states,
if failure_first {
vec![PlaylistUpdateState::Failure, weaker]
} else {
vec![weaker, PlaylistUpdateState::Failure]
}
);
let mut next = processing_context(temp.path());
next.run_id = "next-successful-run".into();
next.execution_order = 2.into();
let success = acquired_result(PlaylistUpdateState::Success);
persist_input_job_result(&next, &success).await;
let replacement = input_cache::load_input_status(&path);
assert_eq!(replacement.last_input_update.unwrap().state, PlaylistUpdateState::Success);
assert_eq!(replacement.clusters, original.clusters);
}
}
}
#[tokio::test]
async fn playlist_update_status_parent_download_persists_staged_own_per_cluster_success_and_failure() {
for fails in [false, true] {
let temp = tempfile::tempdir().unwrap();
let mut responses = fixture_responses([1, 0, 0]);
if fails {
responses.insert("get_live_streams".into(), "not JSON".into());
}
let server = TestXtreamServer::start_with_responses(responses);
let events = CollectSink::default();
let ctx = processing_context_with_events(temp.path(), events.clone());
let parent_file = temp.path().join("parent.m3u");
std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap();
let parent = Arc::new(ConfigInput {
id: 7,
name: "parent".intern(),
input_type: InputType::M3u,
url: parent_file.to_string_lossy().into_owned(),
enabled: true,
..ConfigInput::default()
});
let mut staged =
(*test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, 0, &[]))
.clone();
staged.id = 8;
staged.name = "own-staged".intern();
staged.input_type = InputType::Staged;
staged.staged_type = shared::model::StagedInputType::Xtream;
staged.staged = Some(tuliprox_core::model::ConfigInputStaged {
for_input: Some(parent.name.clone()),
clusters: ClusterFlags::Live,
});
staged.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto {
skip_vod: true,
skip_series: true,
update_quality: ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 },
..ConfigInputOptionsDto::default()
}));
staged.resolve_staged_download_type(); // Same resolver used when SourcesConfig is built.
let staged = Arc::new(staged);
ctx.config.sources.store(Arc::new(SourcesConfig {
inputs: vec![parent.clone(), staged.clone()],
..SourcesConfig::default()
}));
let staged_path = input_storage_path(&ctx, &staged).await;
// Referencing or directly skipping a staged input is not an acquisition.
let skipped = download_input(&ctx, &staged, false).await;
assert!(skipped.input_telemetry.is_none());
assert!(input_cache::load_input_status(&staged_path).last_input_update.is_none());
assert!(staged_completion::input_completions(&events, staged.id).is_empty());
let result = download_input(&ctx, &parent, false).await;
let status = input_cache::load_input_status(&staged_path);
assert!(!status.clusters.contains_key("default"));
assert_eq!(status.clusters.len(), 1);
let completion =
status.last_input_update.expect("actual indirect staged acquisition has its own completion");
assert_eq!(
completion.state,
if fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success }
);
assert!(completion.timestamp > 0);
let completions = staged_completion::input_completions(&events, staged.id);
assert_eq!(completions.len(), 1);
assert_eq!(completions[0].state, Some(completion.state));
assert_eq!(completions[0].run_id, Some(ctx.run_id.clone()));
assert_eq!(completions[0].execution_order, Some(ctx.execution_order));
let snapshot = status.clusters["live"].last_update.unwrap();
assert_eq!(snapshot.source, Some(PlaylistUpdateDataSource::Provider));
assert_eq!(snapshot.policy, None);
if fails {
assert_eq!(snapshot.quality_guard_threshold, Some(95));
assert_eq!(snapshot.quality, None);
} else {
assert_eq!(snapshot.quality_guard_threshold, None, "the evaluation already owns its threshold");
let quality = snapshot.quality.unwrap();
assert_eq!(quality.threshold, 95);
assert_eq!(quality.decision, PersistedPlaylistUpdateQualityDecision::Accepted);
assert_eq!(quality.achieved_quality, None, "bootstrap has no invented Quality");
}
assert_eq!(result.errors.is_empty(), !fails);
let parent_path = input_storage_path(&ctx, &parent).await;
assert_ne!(parent_path, staged_path);
assert!(
input_cache::load_input_status(&parent_path).last_input_update.is_none(),
"staged result does not invent a parent job completion"
);
if !fails {
let bytes = std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap();
let reuse = download_input(&ctx, &parent, false).await;
assert!(reuse.input_telemetry.is_none());
let mut staged_reuse = download_input(&ctx, &staged, true).await;
assert!(staged_reuse.input_telemetry.is_none());
super::super::input_status::complete_staged_input(&ctx, &staged, &mut staged_reuse).await;
assert_eq!(std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap(), bytes);
assert_eq!(staged_completion::input_completions(&events, staged.id), completions);
}
let requests = server.finish();
assert_eq!(
requests.iter().filter(|action| action.as_str() == "get_live_streams").count(),
1,
"no extra staged acquisition on parent reuse"
);
}
}
#[tokio::test]
async fn playlist_update_status_completion_replaces_one_result_without_touching_cache_or_cluster_snapshots() {
let temp = tempfile::tempdir().unwrap();
let ctx = processing_context(temp.path());
let input_name: Arc<str> = "result-input".intern();
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input_name).await;
let mut original = input_cache::InputStatus::default();
input_cache::update_cluster_status(&mut original, "default", input_cache::ClusterState::Ok);
original.clusters.get_mut("default").unwrap().timestamp = 1; // Expired cache must stay expired.
input_cache::replace_cluster_snapshot(
&mut original,
"default",
PersistedPlaylistUpdateClusterSnapshot {
source: Some(PlaylistUpdateDataSource::Cache),
..PersistedPlaylistUpdateClusterSnapshot::default()
},
);
input_cache::save_input_status(&path, &original);
for state in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial, PlaylistUpdateState::Failure] {
persist_input_completion(&ctx, 7, &input_name, state).await;
let reloaded = input_cache::load_input_status(&path);
let result = reloaded.last_input_update.unwrap();
assert_eq!(result.state, state);
assert!(result.timestamp > 1);
assert_eq!(reloaded.clusters, original.clusters);
assert!(!input_cache::is_cache_valid(&reloaded, "default", 3600));
let json: serde_json::Value =
serde_json::from_slice(&std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap()).unwrap();
assert!(json["last_input_update"].is_object());
assert_eq!(json.as_object().unwrap().len(), 2, "only clusters + one result, no history");
}
}
#[tokio::test]
async fn playlist_update_status_in_run_reuse_preserves_original_partial_result_and_timestamp() {
let temp = tempfile::tempdir().unwrap();
let ctx = processing_context(temp.path());
let mut result = super::super::playlist_update_run_result(7, InputJobState::Ready, vec![], true);
result.input_telemetry = Some(PlaylistUpdateInputTelemetry {
refresh_policy: InputRefreshPolicy::NORMAL,
source: Some(PlaylistUpdateDataSource::Provider),
clusters: Vec::new(),
});
persist_input_job_result(&ctx, &result).await;
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await;
let mut original = input_cache::load_input_status(&path);
assert_eq!(original.last_input_update.unwrap().state, PlaylistUpdateState::Partial);
original.last_input_update.as_mut().unwrap().timestamp = 123;
input_cache::save_input_status(&path, &original);
let before = std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap();
result.had_quality_rejections = false;
result.input_telemetry = None; // Same existing absence of acquisition facts as a real in-run reuse.
persist_input_job_result(&ctx, &result).await;
assert_eq!(std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap(), before);
result.state = InputJobState::Failed; // A genuine technical completion must still replace the prior result.
persist_input_job_result(&ctx, &result).await;
assert_eq!(
input_cache::load_input_status(&path).last_input_update.unwrap().state,
PlaylistUpdateState::Failure
);
}
#[tokio::test]
async fn playlist_update_status_completion_covers_every_input_type_without_artificial_clusters() {
let temp = tempfile::tempdir().unwrap();
let ctx = processing_context(temp.path());
for (id, input_type) in [
InputType::Xtream,
InputType::XtreamBatch,
InputType::Stalker,
InputType::StalkerBatch,
InputType::M3u,
InputType::M3uBatch,
InputType::Library,
InputType::Plex,
InputType::Emby,
InputType::Jellyfin,
InputType::Staged,
]
.into_iter()
.enumerate()
{
let input = ConfigInput {
id: u16::try_from(id).unwrap(),
input_type,
name: format!("input-{id}").intern(),
..ConfigInput::default()
};
let result = panicked_input_job(id, &input);
persist_input_job_result(&ctx, &result).await;
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await;
let persisted = input_cache::load_input_status(&path);
assert!(persisted.clusters.is_empty());
assert!(
matches!(persisted.last_input_update, Some(PersistedPlaylistUpdateInputResult {state: PlaylistUpdateState::Failure, timestamp}) if timestamp > 0)
);
}
}
}
@@ -0,0 +1,111 @@
//! M3U's document adapter at the existing locked input acquisition boundary.
use super::ingest::{cluster_is_configured, PIPELINE_TRANSPARENCY_CLUSTERS};
use shared::model::{M3uPlaylistItem, UpdateQualityPolicy};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
use tuliprox_core::model::{evaluate_update_quality, AppConfig, ClusterForceUpdate, ConfigInput};
use tuliprox_iptv::provider::PlaylistFetch;
use tuliprox_repository::{get_input_m3u_playlist_file_path, get_input_storage_path, load_input_m3u_playlist};
/// Called while the existing per-input lock protects baseline, decision and subsequent persistence.
pub(super) async fn effective_m3u_fetch(
config: &Arc<AppConfig>,
input: &ConfigInput,
policy: UpdateQualityPolicy,
mut fetch: PlaylistFetch,
) -> PlaylistFetch {
if !fetch.is_ok() {
return fetch;
}
// Match the existing M3U writer's last-wins primary key population. This also
// keeps the target source and raw group catalogs consistent with its BTree.
let mut winners = HashMap::new();
for (group_index, group) in fetch.groups.iter_mut().enumerate() {
for item in std::mem::take(&mut group.channels) {
winners.insert(M3uPlaylistItem::from(&item).provider_id, (group_index, item));
}
}
let mut winners: Vec<_> = winners.into_values().collect();
winners.sort_by_key(|(group, item)| (*group, item.header.source_ordinal));
for (group_index, item) in winners {
fetch.groups[group_index].channels.push(item);
}
fetch.groups.retain(|group| !group.channels.is_empty());
if policy == UpdateQualityPolicy::Enforce
&& input.options.as_ref().is_none_or(|options| {
PIPELINE_TRANSPARENCY_CLUSTERS.into_iter().all(|cluster| options.update_quality.threshold(cluster) == 0)
})
{
return fetch;
}
let needs_previous = PIPELINE_TRANSPARENCY_CLUSTERS.into_iter().any(|cluster| {
!cluster_is_configured(input, cluster)
|| (policy == UpdateQualityPolicy::Enforce
&& input.options.as_ref().is_some_and(|options| options.update_quality.threshold(cluster) > 0))
});
let previous = if needs_previous {
let storage_dir = config.config.load().storage_dir.clone();
let loaded = async {
let storage = get_input_storage_path(&input.name, &storage_dir).await.map_err(|error| {
shared::error::TuliproxError::RepositoryM3u(format!("Cannot read M3U baseline: {error}"))
})?;
load_input_m3u_playlist(config, &get_input_m3u_playlist_file_path(&storage, &input.name)).await
}
.await;
match loaded {
Ok(groups) => groups,
Err(error) => return PlaylistFetch::failed(error),
}
} else {
Vec::new()
};
let mut retained = Vec::new();
for cluster in PIPELINE_TRANSPARENCY_CLUSTERS {
if !cluster_is_configured(input, cluster) {
retained.push(cluster);
continue;
}
let threshold = input.options.as_ref().map_or(0, |options| options.update_quality.threshold(cluster));
let candidate_count =
fetch.groups.iter().filter(|group| group.xtream_cluster == cluster).map(|group| group.channels.len()).sum();
let current_count = previous
.iter()
.filter(|group| group.xtream_cluster == cluster)
.map(|group| group.channels.len())
.reduce(|total, count| total + count);
if policy == UpdateQualityPolicy::Bypass {
fetch.force_updates.push(ClusterForceUpdate { cluster, candidate_count, configured_threshold: threshold });
continue;
}
let decision = evaluate_update_quality(current_count, candidate_count, threshold);
if let Some(rejection) = decision.rejection(cluster) {
fetch.quality_rejections.push(rejection);
retained.push(cluster);
} else if let Some(acceptance) = decision.acceptance(cluster) {
fetch.quality_acceptances.push(acceptance);
}
}
// Quality decisions must not reorder accepted candidate groups. Append retained
// groups in their existing persisted order, without regrouping either source.
fetch.groups.retain(|group| !retained.contains(&group.xtream_cluster));
fetch.groups.extend(previous.into_iter().filter(|group| retained.contains(&group.xtream_cluster)));
// Retention can combine an old cluster with a newly reclassified primary key.
// The existing document BTree cannot represent both; fail before publication
// instead of losing a retained row or exposing different target/catalog facts.
let mut keys = HashSet::new();
if fetch
.groups
.iter()
.flat_map(|group| &group.channels)
.any(|item| !keys.insert(M3uPlaylistItem::from(item).provider_id))
{
return PlaylistFetch::failed(shared::error::TuliproxError::RepositoryM3u(
"Conflicting M3U keys across retained and candidate clusters".to_owned(),
));
}
fetch
}
@@ -0,0 +1,520 @@
mod m3u_update_quality {
use super::*;
use std::fmt::Write;
use tuliprox_repository::{load_input_m3u_playlist, raw_group_catalog_path, RawInputGroupCatalog};
fn document(prefix: &str, counts: [usize; 3]) -> String {
let mut text = String::from("#EXTM3U\n");
for (cluster, count) in XTREAM_CLUSTER.into_iter().zip(counts) {
for index in 0..count {
let (kind, group, title) = match cluster {
XtreamCluster::Live => ("live", format!("{prefix}-live"), format!("Channel {index}")),
XtreamCluster::Video => ("movie", format!("{prefix}-vod"), format!("Movie {index}")),
XtreamCluster::Series => {
("series", format!("{prefix}-show-{index}"), format!("Show {index} S01E01"))
}
};
let id = match cluster {
XtreamCluster::Live => 1000,
XtreamCluster::Video => 2000,
XtreamCluster::Series => 3000,
} + index;
writeln!(text, "#EXTGRP:{prefix}-{cluster}\n#EXTINF:-1 xui-id=\"{id}\" tvg-type=\"{kind}\" group-title=\"{group}\",{title}\nhttp://stream.example/{prefix}/{cluster}/{id}").unwrap();
}
}
text
}
async fn fixture(
root: &Path,
previous: Option<[usize; 3]>,
thresholds: [u8; 3],
) -> (PlaylistProcessingContext<shared::model::NoopSink>, Arc<ConfigInput>) {
let ctx = processing_context(root);
let input = Arc::new(ConfigInput {
id: 27,
name: "m3u-quality".intern(),
input_type: InputType::M3u,
enabled: true,
url: root.join("provider.m3u").to_string_lossy().into_owned(),
options: Some(ConfigInputOptions::from(&ConfigInputOptionsDto {
update_quality: ConfigInputUpdateQualityDto {
live: thresholds[0],
vod: thresholds[1],
series: thresholds[2],
},
..ConfigInputOptionsDto::default()
})),
..ConfigInput::default()
});
if let Some(counts) = previous {
tokio::fs::write(&input.url, document("previous", counts)).await.unwrap();
let (groups, errors) =
tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), &input)
.await;
assert!(errors.is_empty());
let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, groups).await;
assert!(error.is_none(), "{error:?}");
}
(ctx, input)
}
async fn run(
ctx: &PlaylistProcessingContext<shared::model::NoopSink>,
input: &Arc<ConfigInput>,
counts: [usize; 3],
) -> InputDownloadResult {
tokio::fs::write(&input.url, document("candidate", counts)).await.unwrap();
download_input(ctx, input, false).await
}
fn counts(groups: &[PlaylistGroup]) -> [usize; 3] {
XTREAM_CLUSTER
.map(|cluster| groups.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.channels.len()).sum())
}
async fn verify_effective(
ctx: &PlaylistProcessingContext<shared::model::NoopSink>,
input: &ConfigInput,
result: &mut InputDownloadResult,
expected: [usize; 3],
) {
assert!(result.errors.is_empty(), "{:?}", result.errors);
assert!(result.storage_error.is_none(), "{:?}", result.storage_error);
assert_eq!(result.job_state(), InputJobState::Ready);
let effective = result.source.take_groups();
assert_eq!(counts(&effective), expected);
let storage = tuliprox_repository::get_input_storage_path(&input.name, &ctx.config.config.load().storage_dir)
.await
.unwrap();
let saved = load_input_m3u_playlist(
&ctx.config,
&tuliprox_repository::get_input_m3u_playlist_file_path(&storage, &input.name),
)
.await
.unwrap();
assert_eq!(counts(&saved), expected);
for cluster in XTREAM_CLUSTER {
let catalog: RawInputGroupCatalog =
serde_json::from_slice(&tokio::fs::read(raw_group_catalog_path(&storage, cluster)).await.unwrap())
.unwrap();
let mut expected_groups: Vec<_> =
effective.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.title.to_string()).collect();
expected_groups.sort();
expected_groups.dedup();
assert_eq!(catalog.groups, expected_groups);
}
}
#[tokio::test]
async fn m3u_update_quality_all_clusters_accepted() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
let mut result = run(&ctx, &input, [9, 11, 10]).await;
assert!(result.quality_rejections.is_empty());
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await;
}
async fn order_candidate(
ctx: &PlaylistProcessingContext<shared::model::NoopSink>,
input: &ConfigInput,
prefix: &str,
) -> tuliprox_iptv::provider::PlaylistFetch {
let mut body = String::from("#EXTM3U\n");
for (id, kind, group) in [
(2000, "movie", "obsolete"),
(2000, "movie", "movies-a"),
(1000, "live", "live-a"),
(2001, "movie", "movies-b"),
(1001, "live", "live-b"),
(3000, "series", "shows-a"),
(3001, "series", "shows-b"),
] {
writeln!(body, "#EXTGRP:{prefix}-{group}\n#EXTINF:-1 tvg-type=\"{kind}\" group-title=\"{prefix}-{group}\",Item {id} S01E01\nhttp://stream.example/{kind}/{id}").unwrap();
}
tokio::fs::write(&input.url, body).await.unwrap();
let (groups, errors) =
tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), input).await;
assert!(errors.is_empty());
assert_eq!(
group_names(&groups),
["obsolete", "movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"]
.map(|name| format!("{prefix}-{name}"))
);
tuliprox_iptv::provider::PlaylistFetch::groups(groups)
}
fn group_names(groups: &[PlaylistGroup]) -> Vec<String> {
groups.iter().map(|group| group.title.to_string()).collect()
}
#[tokio::test]
async fn m3u_update_quality_preserves_candidate_group_order_when_disabled() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await;
let candidate = order_candidate(&ctx, &input, "candidate").await;
let effective = super::super::super::m3u_quality::effective_m3u_fetch(
&ctx.config,
&input,
shared::model::UpdateQualityPolicy::Enforce,
candidate,
)
.await;
assert!(effective.errors.is_empty());
assert!(effective.quality_acceptances.is_empty() && effective.quality_rejections.is_empty());
assert_eq!(
group_names(&effective.groups),
["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"]
.map(|name| format!("candidate-{name}"))
);
assert_eq!(counts(&effective.groups), [2, 2, 2], "disabled quality processing must collapse duplicate keys");
}
#[tokio::test]
async fn m3u_update_quality_preserves_candidate_group_order_when_all_clusters_accepted() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([2, 2, 2]), [100, 100, 100]).await;
let candidate = order_candidate(&ctx, &input, "candidate").await;
let effective = super::super::super::m3u_quality::effective_m3u_fetch(
&ctx.config,
&input,
shared::model::UpdateQualityPolicy::Enforce,
candidate,
)
.await;
assert!(effective.errors.is_empty() && effective.quality_rejections.is_empty());
assert_eq!(effective.quality_acceptances.len(), 3);
assert_eq!(
group_names(&effective.groups),
["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"].map(|name| format!("candidate-{name}"))
);
}
#[tokio::test]
async fn m3u_update_quality_mixed_fallback_preserves_relative_group_order() {
for _ in 0..3 {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), None, [100, 100, 100]).await;
let mut previous = order_candidate(&ctx, &input, "previous").await.groups;
previous.retain(|group| group.title.as_ref() != "previous-obsolete");
let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, previous).await;
assert!(error.is_none());
let mut previous = load_input_playlist(&ctx.config, &input, None).await.unwrap();
let previous = previous.take_groups();
assert_eq!(
previous
.iter()
.filter(|group| group.xtream_cluster == XtreamCluster::Series)
.map(|group| group.title.to_string())
.collect::<Vec<_>>(),
["previous-shows-a", "previous-shows-b"]
);
let mut candidate = order_candidate(&ctx, &input, "candidate").await;
candidate.groups.retain(|group| group.title.as_ref() != "candidate-shows-b");
let effective = super::super::super::m3u_quality::effective_m3u_fetch(
&ctx.config,
&input,
shared::model::UpdateQualityPolicy::Enforce,
candidate,
)
.await;
assert!(effective.errors.is_empty());
assert_eq!(effective.quality_rejections.len(), 1);
assert_eq!(effective.quality_rejections[0].cluster, XtreamCluster::Series);
assert_eq!(
group_names(&effective.groups),
[
"candidate-movies-a",
"candidate-live-a",
"candidate-movies-b",
"candidate-live-b",
"previous-shows-a",
"previous-shows-b"
]
);
}
}
#[tokio::test]
async fn m3u_update_quality_retains_only_rejected_cluster() {
for disk in [false, true] {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
let mut config = (**ctx.config.config.load()).clone();
config.disk_based_processing = disk;
ctx.config.config.store(Arc::new(config));
let mut result = run(&ctx, &input, [9, 11, 8]).await;
assert_eq!(result.update_state(), PlaylistUpdateState::Partial);
assert_eq!(result.quality_rejections.len(), 1);
assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series);
verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await;
}
}
#[tokio::test]
async fn m3u_update_quality_zero_candidate_retains_previous_cluster() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
let mut result = run(&ctx, &input, [10, 10, 0]).await;
assert_eq!(result.quality_rejections[0].candidate_count, 0);
verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await;
}
#[tokio::test]
async fn m3u_update_quality_disabled() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [0, 0, 0]).await;
let mut result = run(&ctx, &input, [1, 1, 0]).await;
assert!(result.quality_rejections.is_empty());
assert!(result
.input_telemetry
.as_ref()
.unwrap()
.clusters
.iter()
.all(|c| c.quality.is_none() && c.threshold.is_none()));
verify_effective(&ctx, &input, &mut result, [1, 1, 0]).await;
}
#[tokio::test]
async fn m3u_update_quality_first_import() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), None, [90, 90, 90]).await;
let mut result = run(&ctx, &input, [1, 1, 1]).await;
assert!(result.quality_rejections.is_empty());
assert!(result.input_telemetry.as_ref().unwrap().clusters.iter().all(|c| c.quality.is_none()
&& c.baseline_count.is_none()
&& c.decision == Some(PlaylistUpdateClusterDecision::Accepted)));
verify_effective(&ctx, &input, &mut result, [1, 1, 1]).await;
}
#[tokio::test]
async fn m3u_update_quality_download_or_parse_failure() {
for body in [None, Some("<html>provider unavailable</html>"), Some("#EXTM3U\n#EXTINF:-1,Truncated\n")] {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
if let Some(body) = body {
tokio::fs::write(&input.url, body).await.unwrap();
} else {
tokio::fs::remove_file(&input.url).await.unwrap();
}
let mut result = download_input(&ctx, &input, false).await;
assert!(!result.errors.is_empty());
assert!(result.quality_rejections.is_empty());
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert_eq!(counts(&saved.take_groups()), [10, 10, 10]);
}
}
#[tokio::test]
async fn m3u_update_quality_empty_bootstrap_and_structural_empty_use_existing_domain() {
for previous in [None, Some([10, 10, 0])] {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), previous, [90, 90, 90]).await;
let mut result = run(&ctx, &input, if previous.is_some() { [10, 10, 0] } else { [0, 0, 0] }).await;
if previous.is_none() {
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
assert_eq!(result.quality_rejections.len(), 3);
} else {
assert_eq!(result.update_state(), PlaylistUpdateState::Partial);
assert_eq!(result.quality_rejections.len(), 1);
assert_eq!(result.quality_rejections[0].current_count, 0);
assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series);
}
}
}
#[tokio::test]
async fn m3u_update_quality_skip_preserves_previous_and_reuse_emits_no_new_facts() {
let temp = tempfile::tempdir().unwrap();
let (ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
Arc::make_mut(&mut input).options.as_mut().unwrap().flags.set(ConfigInputFlags::SkipSeries);
let mut result = run(&ctx, &input, [10, 10, 0]).await;
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
assert!(result.quality_rejections.is_empty());
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert_eq!(counts(&result.source.take_groups()), [10, 10, 0]);
assert_eq!(counts(&saved.take_groups()), [10, 10, 10]);
let telemetry = result.input_telemetry.unwrap();
assert!(!telemetry.clusters.iter().find(|c| c.cluster == XtreamCluster::Series).unwrap().requested);
tokio::fs::remove_file(&input.url).await.unwrap();
let reused = download_input(&ctx, &input, false).await;
assert!(reused.errors.is_empty());
assert!(reused.input_telemetry.is_none());
}
#[tokio::test]
async fn m3u_update_quality_force_bypasses_guard_and_cache_without_inventing_quality() {
for threshold in [0, 95] {
for policy in [InputRefreshPolicy::REFRESH, InputRefreshPolicy::FORCE] {
let temp = tempfile::tempdir().unwrap();
let (mut ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [threshold; 3]).await;
Arc::make_mut(&mut input).cache_duration_seconds = 3_600;
let storage = input_storage_path(&ctx, &input).await;
let mut status = input_cache::InputStatus::default();
input_cache::update_cluster_status(&mut status, "default", input_cache::ClusterState::Ok);
input_cache::save_input_status(&storage, &status);
set_refresh_policy(&mut ctx, &input, policy);
let mut result = run(&ctx, &input, [1, 1, 1]).await;
let rejected = threshold > 0 && policy == InputRefreshPolicy::REFRESH;
assert_eq!(
result.update_state(),
if rejected { PlaylistUpdateState::Partial } else { PlaylistUpdateState::Success }
);
assert_eq!(counts(&result.source.take_groups()), if rejected { [10; 3] } else { [1; 3] });
let telemetry = result.input_telemetry.as_ref().unwrap();
assert_eq!(telemetry.refresh_policy, policy);
assert!(telemetry
.clusters
.iter()
.all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider)));
let saved = input_cache::load_input_status(&storage);
for cluster in &telemetry.clusters {
if policy == InputRefreshPolicy::FORCE {
assert_eq!(cluster.decision, Some(PlaylistUpdateClusterDecision::Accepted));
assert_eq!((cluster.baseline_count, cluster.quality), (None, None));
assert_eq!((cluster.candidate_count, cluster.active_count), (Some(1), Some(1)));
let snapshot = saved.clusters[cluster.cluster.as_ref()].last_update.unwrap();
assert_eq!(snapshot.quality, None);
assert_eq!(snapshot.quality_guard_threshold, Some(threshold));
assert_eq!(snapshot.policy, Some(policy));
}
}
let reused = download_input(&ctx, &input, false).await;
assert!(reused.input_telemetry.is_none(), "in-run reuse must not create another Force acquisition");
}
}
}
#[tokio::test]
async fn m3u_update_quality_force_keeps_parse_and_empty_publication_errors() {
for body in ["#EXTM3U\n", "#EXTM3U\n#EXTINF:-1,Truncated\n"] {
let temp = tempfile::tempdir().unwrap();
let (mut ctx, input) = fixture(temp.path(), Some([10; 3]), [95; 3]).await;
set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE);
tokio::fs::write(&input.url, body).await.unwrap();
let mut result = download_input(&ctx, &input, false).await;
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
assert!(result.quality_rejections.is_empty());
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert_eq!(counts(&saved.take_groups()), [10; 3]);
}
}
#[tokio::test]
async fn m3u_update_quality_target_processing_uses_retained_and_accepted_groups() {
for policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::FORCE] {
let temp = tempfile::tempdir().unwrap();
let (mut ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
set_refresh_policy(&mut ctx, &input, policy);
tokio::fs::write(&input.url, document("candidate", [10, 0, 10])).await.unwrap();
let output = temp.path().join("client.m3u");
let mut target = ConfigTarget::from(&ConfigTargetDto {
id: 1,
name: "m3u-target".into(),
enabled: true,
..ConfigTargetDto::default()
});
target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into();
target.output = vec![TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput {
filename: Some(output.to_string_lossy().into_owned()),
include_type_in_url: false,
mask_redirect_url: false,
filter: None,
})];
ctx.config.sources.store(Arc::new(SourcesConfig {
inputs: vec![input.clone()],
sources: vec![ConfigSource { inputs: vec![input.name.clone()], targets: vec![Arc::new(target)] }],
..SourcesConfig::default()
}));
let (_, targets, errors) = process_source(0, Arc::new(ctx)).await;
assert!(errors.is_empty(), "{errors:?}");
assert_eq!(targets.len(), 1);
let published = tokio::fs::read_to_string(output).await.unwrap();
assert!(published.contains("/candidate/live/") && published.contains("/candidate/series/"), "{published}");
assert_eq!(published.contains("/previous/video/"), policy == InputRefreshPolicy::NORMAL, "{published}");
assert!(!published.contains("/previous/live/") && !published.contains("/candidate/video/"));
}
}
#[tokio::test]
async fn m3u_update_quality_raw_group_catalog_uses_effective_state() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
let mut result = run(&ctx, &input, [10, 10, 0]).await;
let storage = input_cache::resolve_input_storage_path(&ctx.config.config.load().storage_dir, &input.name).await;
let status = input_cache::load_input_status(&storage);
for cluster in XTREAM_CLUSTER {
let saved = &status.clusters[cluster.as_ref()];
let rejected = cluster == XtreamCluster::Series;
assert_eq!(
saved.status,
if rejected { input_cache::ClusterState::Failed } else { input_cache::ClusterState::Ok }
);
let snapshot = saved.last_update.unwrap();
assert_eq!(
snapshot.quality.unwrap().decision,
if rejected {
PersistedPlaylistUpdateQualityDecision::Rejected
} else {
PersistedPlaylistUpdateQualityDecision::Accepted
}
);
assert_eq!(snapshot.active_count, Some(10));
}
assert_eq!(status.clusters["default"].status, input_cache::ClusterState::Failed);
let series: RawInputGroupCatalog = serde_json::from_slice(
&tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Series)).await.unwrap(),
)
.unwrap();
assert_eq!(series.groups, ["previous-series"]);
verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await;
}
#[tokio::test]
async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [100, 100, 0]).await;
let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]);
tokio::fs::write(&input.url, candidate).await.unwrap();
let mut result = download_input(&ctx, &input, false).await;
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last")));
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert_eq!(counts(&saved.take_groups()), [1, 1, 0]);
}
#[tokio::test]
async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population_when_disabled() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await;
let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]);
tokio::fs::write(&input.url, candidate).await.unwrap();
let mut result = download_input(&ctx, &input, false).await;
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last")));
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert_eq!(counts(&saved.take_groups()), [1, 1, 0]);
}
#[tokio::test]
async fn m3u_update_quality_conflicting_retained_key_cannot_diverge_from_storage() {
let temp = tempfile::tempdir().unwrap();
let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [90, 0, 0]).await;
let candidate = document("candidate", [0, 1, 0]).replace("2000", "1000");
tokio::fs::write(&input.url, candidate).await.unwrap();
let mut result = download_input(&ctx, &input, false).await;
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
assert!(!result.errors.is_empty());
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert_eq!(counts(&saved.take_groups()), [1, 1, 0]);
let storage = input_storage_path(&ctx, &input).await;
let catalog: RawInputGroupCatalog = serde_json::from_slice(
&tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Live)).await.unwrap(),
)
.unwrap();
assert_eq!(catalog.groups, ["previous-live"]);
}
}
@@ -0,0 +1,198 @@
use super::{process_sources, MetadataUpdateSink, PlaylistProcessingContext};
use shared::{
error::TuliproxError,
model::{
EventMessage, EventSink, LibraryScanResult, PlaylistUpdateProgressDetail, PlaylistUpdateProgressEvent,
PlaylistUpdateState, SourceStats,
},
};
use std::{future::Future, io};
use tuliprox_core::model::UpdateGuard;
use tuliprox_library::library::LibraryProcessor;
/// Bulk and scheduled runs read the existing catalog. Only an explicit card request scans.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) enum LibraryUpdateMode {
#[default]
ExistingCatalog,
Rescan {
input_id: u16,
},
}
impl LibraryUpdateMode {
pub(crate) fn reloads_input(self, input_id: u16) -> bool { self == Self::Rescan { input_id } }
}
/// The rebuild is not even started until the existing scanner reports complete success.
async fn rebuild_after_scan<T>(
scan: impl Future<Output = io::Result<LibraryScanResult>>,
report: impl FnOnce(LibraryScanResult),
rebuild: impl FnOnce() -> T,
) -> io::Result<T> {
let result = scan.await?;
let complete = result.errors == 0;
report(result);
if !complete {
return Err(io::Error::other("Library rescan reported incomplete input data"));
}
Ok(rebuild())
}
pub(super) async fn process_manual_update<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
ctx: &PlaylistProcessingContext<E, M>,
update_guard: Option<&UpdateGuard>,
) -> (Vec<SourceStats>, Vec<TuliproxError>) {
let LibraryUpdateMode::Rescan { input_id } = ctx.library_update_mode else {
return process_sources(ctx).await;
};
let progress = |message: &str, detail| {
ctx.events.emit(EventMessage::PlaylistUpdateProgress(
PlaylistUpdateProgressEvent::for_run_input(
ctx.run_id.clone(),
ctx.execution_order,
input_id,
"Library rescan",
message,
)
.with_detail(detail),
));
};
progress(
"Library rescan started; selected target rebuild is waiting",
PlaylistUpdateProgressDetail::LibraryRescanStarted,
);
let result = async {
// The worker already owns the playlist permit. Never wait for a second lock:
// a concurrent standalone scan makes this request fail, without a lock cycle.
let _library_permit = update_guard
.and_then(UpdateGuard::try_library)
.ok_or_else(|| io::Error::other("Library scan is already running or its guard is unavailable"))?;
let (library, metadata, storage_dir) = {
let config = ctx.config.config.load();
let library = config
.library
.as_ref()
.filter(|library| library.enabled)
.ok_or_else(|| io::Error::other("Library is not enabled"))?
.clone();
(library, config.metadata_update.clone(), config.storage_dir.clone())
};
let processor = LibraryProcessor::new(library, metadata.as_ref(), ctx.client.clone(), &storage_dir);
let rebuild = rebuild_after_scan(
processor.scan_for_target_rebuild(),
|result| {
if result.errors > 0 {
log::error!("Library rescan failed for input {input_id}: incomplete scan; {result:?}");
}
ctx.events.emit(EventMessage::PlaylistUpdateProgress(
PlaylistUpdateProgressEvent::for_run_input(
ctx.run_id.clone(),
ctx.execution_order,
input_id,
"Library rescan",
"Library scan result",
)
.with_library_scan_result(result)
.with_detail(PlaylistUpdateProgressDetail::LibraryScanResult),
));
},
|| {
progress(
"Library rescan completed; loading input data and rebuilding selected targets",
PlaylistUpdateProgressDetail::LibraryRescanCompleted,
);
process_sources(ctx)
},
)
.await?;
// Retain the existing library permit until the catalog-dependent rebuild finishes.
Ok::<_, io::Error>(rebuild.await)
}
.await;
match result {
Ok(result) => result,
Err(error) => {
log::error!("Library rescan failed for input {input_id}: {error}");
if let Some(input) = ctx.config.get_input_by_id(input_id) {
super::input_status::persist_input_completion(ctx, input.id, &input.name, PlaylistUpdateState::Failure)
.await;
}
ctx.events.emit(EventMessage::PlaylistUpdateProgress(
PlaylistUpdateProgressEvent::input_completed(
ctx.run_id.clone(),
ctx.execution_order,
input_id,
PlaylistUpdateState::Failure,
"Library rescan",
"Library rescan failed; selected targets were not rebuilt",
)
.with_detail(PlaylistUpdateProgressDetail::LibraryRescanFailed),
));
(Vec::new(), vec![TuliproxError::RepositoryLibrary(format!("Library rescan failed: {error}"))])
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::Cell;
fn scan_result(errors: usize) -> LibraryScanResult {
LibraryScanResult {
files_scanned: 0,
groups_scanned: 0,
files_added: 0,
files_updated: 0,
files_removed: 0,
errors,
}
}
#[tokio::test]
async fn manual_update_library_rebuild_waits_for_successful_scan() {
let completed = Cell::new(false);
let scan = async {
tokio::task::yield_now().await;
completed.set(true);
Ok(scan_result(0))
};
let rebuilt = rebuild_after_scan(
scan,
|_| {},
|| {
assert!(completed.get());
"common target pipeline"
},
)
.await;
assert_eq!(rebuilt.unwrap(), "common target pipeline");
}
#[tokio::test]
async fn manual_update_library_errors_and_partial_scan_never_start_target_rebuild() {
for result in [Err(io::Error::other("scan failed")), Ok(LibraryScanResult { errors: 1, ..scan_result(0) })] {
let rebuilt = Cell::new(false);
let reported = std::cell::RefCell::new(None);
let expected = result.as_ref().ok().cloned();
assert!(rebuild_after_scan(
async { result },
|scan| *reported.borrow_mut() = Some(scan),
|| rebuilt.set(true)
)
.await
.is_err());
assert_eq!(*reported.borrow(), expected);
assert!(!rebuilt.get());
}
}
#[test]
fn manual_update_library_bulk_reuses_catalog_without_scan_or_policy_override() {
assert_eq!(LibraryUpdateMode::default(), LibraryUpdateMode::ExistingCatalog);
assert!(!LibraryUpdateMode::ExistingCatalog.reloads_input(2));
assert!(LibraryUpdateMode::Rescan { input_id: 2 }.reloads_input(2));
assert!(!LibraryUpdateMode::Rescan { input_id: 2 }.reloads_input(3));
}
}
@@ -0,0 +1,725 @@
mod manual_update_integration {
use super::*;
use shared::model::{ConfigRenameDto, ItemField, LibraryConfigDto, LibraryContentType, LibraryScanResult};
use tuliprox_core::model::{LibraryConfig, LibraryScanDirectory, M3uTargetOutput, MetadataUpdateConfig};
async fn library_context(root: &Path, order: ProcessingOrder) -> PlaylistProcessingContext<CollectSink> {
let ctx = processing_context_with_events(root, CollectSink::default());
let media = root.join("media");
tokio::fs::create_dir_all(&media).await.unwrap();
tokio::fs::write(media.join("Library Movie (2025).mp4"), b"fixture").await.unwrap();
let companion_path = root.join("companion.m3u");
tokio::fs::write(&companion_path, "#EXTM3U\n#EXTINF:-1,Companion\nhttp://stream.example/companion\n")
.await
.unwrap();
let mut config = (**ctx.config.config.load()).clone();
let mut library = LibraryConfig::from(&LibraryConfigDto { enabled: true, ..LibraryConfigDto::default() });
library.metadata.fallback_to_filename = true;
library.thumbnails.enabled = false;
library.scan_directories = vec![LibraryScanDirectory {
enabled: true,
path: media.to_string_lossy().into_owned(),
content_type: LibraryContentType::Movie,
recursive: true,
}];
config.library = Some(library);
let mut metadata = MetadataUpdateConfig::default();
metadata.ffprobe.enabled = false;
metadata.tmdb.enabled = false;
config.metadata_update = Some(metadata);
ctx.config.config.store(Arc::new(config));
let library_input = Arc::new(ConfigInput {
id: 2,
name: "local".intern(),
input_type: InputType::Library,
enabled: true,
cache_duration_seconds: 3600,
..ConfigInput::default()
});
let companion = Arc::new(ConfigInput {
id: 3,
name: "companion".intern(),
input_type: InputType::M3u,
enabled: true,
url: companion_path.to_string_lossy().into_owned(),
..ConfigInput::default()
});
let mut target = ConfigTarget::from(&ConfigTargetDto {
id: 20,
enabled: true,
name: "selected".into(),
processing_order: order,
rename: Some(vec![ConfigRenameDto {
field: ItemField::Name,
pattern: "^MAPPED$".into(),
new_name: "RENAMED".into(),
t_pattern: None,
}]),
..ConfigTargetDto::default()
});
target.output = vec![TargetOutput::M3u(M3uTargetOutput {
filename: Some(root.join("selected.m3u").to_string_lossy().into_owned()),
include_type_in_url: false,
mask_redirect_url: false,
filter: None,
})];
target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into();
let mapped = super::super::mapping_stage::build_target(
vec![super::super::mapping_stage::build_mapping("manual", MappingStage::Processing, r#"@Name = "MAPPED""#)],
false,
);
target.mapping = mapped.mapping;
let mut unselected = target.clone();
unselected.id = 21;
unselected.name = "unselected".into();
unselected.output = vec![TargetOutput::M3u(M3uTargetOutput {
filename: Some(root.join("unselected.m3u").to_string_lossy().into_owned()),
include_type_in_url: false,
mask_redirect_url: false,
filter: None,
})];
ctx.config.sources.store(Arc::new(SourcesConfig {
inputs: vec![library_input.clone(), companion.clone()],
sources: vec![ConfigSource {
inputs: vec![library_input.name.clone(), companion.name.clone()],
targets: vec![Arc::new(target), Arc::new(unselected)],
}],
..SourcesConfig::default()
}));
ctx
}
fn rescan_run(ctx: &PlaylistProcessingContext<CollectSink>, guard: UpdateGuard) -> ProcessingRun<CollectSink> {
ProcessingRun::for_run(
"manual-library-run".into(),
ctx.client.clone(),
ctx.config.clone(),
Arc::new(ProcessTargets {
enabled: true,
inputs: vec![2, 3],
targets: vec![20],
target_names: vec!["selected".into()],
}),
ctx.events.clone(),
)
.with_update_guard(guard)
.with_manual_input_update(Some(InputUpdateRequest { input_id: 2, action: InputUpdateAction::Rescan }))
}
fn scan_results(ctx: &PlaylistProcessingContext<CollectSink>) -> Vec<LibraryScanResult> {
ctx.events
.0
.lock()
.unwrap()
.iter()
.filter_map(|event| match event {
EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) => {
if progress.library_scan_result.is_some() {
assert_eq!(
progress.run_id.as_ref().map(PlaylistUpdateRunId::as_ref),
Some("manual-library-run")
);
assert!(progress.execution_order.is_some());
}
progress.library_scan_result.clone()
}
_ => None,
})
.collect()
}
fn assert_run_state(ctx: &PlaylistProcessingContext<CollectSink>, state: PlaylistUpdateState) {
let events = ctx.events.0.lock().unwrap();
assert!(
events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == state)),
"{events:?}"
);
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) && progress.state == Some(state))), "{events:?}");
}
#[tokio::test]
async fn manual_update_library_empty_rescan_persists_zero_to_zero_and_nonzero_to_zero_with_companion() {
for disk in [false, true] {
for previous_population in [false, true] {
for companion in [false, true] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
let mut config = (**ctx.config.config.load()).clone();
config.disk_based_processing = disk;
ctx.config.config.store(Arc::new(config));
if !companion {
let mut sources = (**ctx.config.sources.load()).clone();
sources.sources[0].inputs.retain(|name| name.as_ref() == "local");
ctx.config.sources.store(Arc::new(sources));
}
if previous_population {
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
ctx.events.0.lock().unwrap().clear();
}
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
for name in std::iter::once("local").chain(companion.then_some("companion")) {
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), name).await;
let status = input_cache::load_input_status(&path);
assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Success, "{name}");
}
assert_eq!(
scan_results(&ctx),
vec![LibraryScanResult {
files_scanned: 0,
groups_scanned: 0,
files_added: 0,
files_updated: 0,
files_removed: usize::from(previous_population),
errors: 0,
}]
);
let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
assert_eq!(output.matches("#EXTINF").count(), usize::from(companion), "{output}");
assert_eq!(output.contains("http://stream.example/companion"), companion);
assert!(!output.contains("Library%20Movie") && !output.contains("file://"));
assert!(!temp.path().join("unselected.m3u").exists());
let input = ctx.config.sources.load().inputs[0].clone();
let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert!(persisted.is_empty(), "Empty Library must replace its old persisted input");
assert!(!ctx.events.0.lock().unwrap().iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.message.contains("playlist is empty"))));
// Same canonical empty input remains readable on subsequent cache/reuse paths.
let first = download_input(&ctx, &input, false).await;
assert!(first.errors.is_empty() && first.storage_error.is_none());
let mut reuse = download_input(&ctx, &input, false).await;
assert_eq!(reuse.job_state(), InputJobState::Ready);
assert_eq!(reuse.input_telemetry, None, "In-run reuse must not emit another acquisition");
assert_eq!(scan_results(&ctx).len(), 1, "Reuse must not manufacture a second rescan");
}
}
}
}
#[tokio::test]
async fn coderabbit_library_unavailable_is_not_an_authoritative_empty_catalog() {
for disabled in [false, true] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
let mut input = (*ctx.config.sources.load().inputs[0]).clone();
input.cache_duration_seconds = 0;
let input = Arc::new(input);
let storage = input_storage_path(&ctx, &input).await;
let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name);
let before = tokio::fs::read(&file).await.unwrap();
let output = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap();
let mut config = (**ctx.config.config.load()).clone();
if disabled {
config.library.as_mut().unwrap().enabled = false;
} else {
config.library = None;
}
ctx.config.config.store(Arc::new(config));
let mut result = download_input(&ctx, &input, false).await;
assert_eq!(result.job_state(), InputJobState::Failed);
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
assert!(!result.authoritative_library);
assert!(!result.errors.is_empty());
assert_eq!(tokio::fs::read(&file).await.unwrap(), before);
assert_eq!(tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(), output);
}
}
#[tokio::test]
async fn coderabbit_library_missing_input_triggers_catalog_recovery_not_empty_success() {
for disk in [false, true] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
let mut config = (**ctx.config.config.load()).clone();
config.disk_based_processing = disk;
ctx.config.config.store(Arc::new(config));
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
let input = ctx.config.sources.load().inputs[0].clone();
let storage = input_storage_path(&ctx, &input).await;
let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name);
tokio::fs::remove_file(&file).await.unwrap();
assert!(load_input_playlist(&ctx.config, &input, None).await.is_err());
let mut recovered = download_input(&ctx, &input, false).await;
assert_eq!(recovered.job_state(), InputJobState::Ready);
assert!(recovered.errors.is_empty() && recovered.storage_error.is_none());
assert!(!recovered.source.is_empty(), "missing cache must be repaired from the populated catalog");
assert!(file.exists());
let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap();
assert!(!persisted.is_empty());
let media_server = ConfigInput { input_type: InputType::Plex, ..(*input).clone() };
assert!(
load_input_playlist(&ctx.config, &media_server, None).await.is_err(),
"the shared loader must not turn a missing Plex cache into a successful empty input"
);
}
}
#[tokio::test]
async fn manual_update_library_scan_errors_keep_real_result_and_block_rebuild() {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
let mut config = (**ctx.config.config.load()).clone();
config.library.as_mut().unwrap().metadata.fallback_to_filename = false;
ctx.config.config.store(Arc::new(config));
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Failure);
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await;
assert_eq!(
input_cache::load_input_status(&path).last_input_update.unwrap().state,
PlaylistUpdateState::Failure
);
assert_eq!(
scan_results(&ctx),
vec![LibraryScanResult {
files_scanned: 1,
groups_scanned: 1,
files_added: 0,
files_updated: 0,
files_removed: 0,
errors: 1,
}]
);
assert!(!temp.path().join("selected.m3u").exists());
}
#[tokio::test]
async fn manual_update_library_persisted_input_status_keeps_failed_companion_separate() {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "companion").await;
assert_eq!(
input_cache::load_input_status(&path).last_input_update.unwrap().state,
PlaylistUpdateState::Success
);
let before = std::fs::read(temp.path().join("selected.m3u")).unwrap();
tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_eq!(
input_cache::load_input_status(&path).last_input_update.unwrap().state,
PlaylistUpdateState::Failure
);
let library_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await;
assert_eq!(
input_cache::load_input_status(&library_path).last_input_update.unwrap().state,
PlaylistUpdateState::Success
);
assert_eq!(std::fs::read(temp.path().join("selected.m3u")).unwrap(), before);
}
#[tokio::test]
async fn manual_update_library_empty_xtream_and_strm_use_existing_publish_and_keep_companions() {
use tuliprox_repository::{xtream_get_file_path, xtream_get_storage_path, BPlusTreeQuery};
for companion in [false, true] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
tokio::fs::write(
temp.path().join("companion.m3u"),
"#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n",
)
.await
.unwrap();
let mut sources = (**ctx.config.sources.load()).clone();
if !companion {
sources.sources[0].inputs.retain(|name| name.as_ref() == "local");
}
let target = Arc::make_mut(&mut sources.sources[0].targets[0]);
target.output.extend([
TargetOutput::Xtream(XtreamTargetOutput {
flags: XtreamTargetFlagsSet::new(),
trakt: None,
filter: None,
}),
TargetOutput::Strm(tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto {
directory: temp.path().join("strm").to_string_lossy().into_owned(),
flat: true,
..shared::model::StrmTargetOutputDto::default()
})),
]);
ctx.config.sources.store(Arc::new(sources));
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
ctx.events.0.lock().unwrap().clear();
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
let config = ctx.config.config.load();
let xtream = xtream_get_storage_path(&config, "selected").unwrap();
for (cluster, expected) in
[(XtreamCluster::Live, 0), (XtreamCluster::Video, usize::from(companion)), (XtreamCluster::Series, 0)]
{
let path = xtream_get_file_path(&xtream, cluster);
if !path.exists() {
assert_eq!(expected, 0);
continue;
}
let mut query = BPlusTreeQuery::<u32, shared::model::XtreamPlaylistItem>::try_new(&path).unwrap();
assert_eq!(query.iter().count(), expected, "{cluster:?}");
}
let entries = strm_artifacts(&temp.path().join("strm")).await;
assert_eq!(entries.len(), usize::from(companion), "{entries:?}");
if companion {
assert!(entries[0].1.contains("http://stream.example/companion"));
}
}
}
#[tokio::test]
async fn manual_update_library_empty_permission_does_not_loosen_other_provider_inputs() {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
for input_type in [InputType::M3u, InputType::Xtream, InputType::Stalker, InputType::Plex] {
let input =
ConfigInput { input_type, name: format!("empty-{input_type:?}").intern(), ..ConfigInput::default() };
let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, Vec::new()).await;
assert!(error.is_some(), "{input_type:?} must retain its generic empty-input guard");
}
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap();
ctx.events.0.lock().unwrap().clear();
tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
{
let events = ctx.events.0.lock().unwrap();
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure)));
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure))));
}
assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap());
// An authoritatively empty Library must not mask a failed companion either.
ctx.events.0.lock().unwrap().clear();
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
{
let events = ctx.events.0.lock().unwrap();
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure)));
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure))));
}
assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap());
}
#[derive(Clone, Copy, Debug)]
enum LibraryFilterBoundary {
Processing,
Persist,
M3u,
Strm,
}
async fn strm_artifacts(root: &Path) -> Vec<(PathBuf, String)> {
let mut directories = vec![root.to_path_buf()];
let mut artifacts = Vec::new();
while let Some(directory) = directories.pop() {
let mut entries = tokio::fs::read_dir(directory).await.unwrap();
while let Some(entry) = entries.next_entry().await.unwrap() {
if entry.file_type().await.unwrap().is_dir() {
directories.push(entry.path());
} else if entry.path().extension().is_some_and(|extension| extension == "strm") {
artifacts.push((entry.path(), tokio::fs::read_to_string(entry.path()).await.unwrap()));
}
}
}
artifacts.sort();
artifacts
}
async fn assert_library_filtered_empty_publication(boundary: LibraryFilterBoundary) {
for disk in [false, true] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
let mut config = (**ctx.config.config.load()).clone();
config.disk_based_processing = disk;
ctx.config.config.store(Arc::new(config));
tokio::fs::write(
temp.path().join("companion.m3u"),
"#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n",
)
.await
.unwrap();
let mut sources = (**ctx.config.sources.load()).clone();
let target = Arc::make_mut(&mut sources.sources[0].targets[0]);
target.rename = None;
target.mapping = Arc::default();
let library_only = get_filter(r#"Name ~ "^Library Movie.*""#, None).unwrap();
if matches!(boundary, LibraryFilterBoundary::Processing) {
target.filter = library_only.clone().into();
}
if matches!(boundary, LibraryFilterBoundary::Persist) {
target.filter.persist = Some(library_only.clone());
}
if let TargetOutput::M3u(output) = &mut target.output[0] {
output.filter = matches!(boundary, LibraryFilterBoundary::M3u).then(|| library_only.clone());
}
let strm_root = temp.path().join("strm");
let mut strm = tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto {
directory: strm_root.to_string_lossy().into_owned(),
flat: true,
cleanup: true,
..shared::model::StrmTargetOutputDto::default()
});
strm.filter = matches!(boundary, LibraryFilterBoundary::Strm).then_some(library_only);
target.output.push(TargetOutput::Strm(strm));
ctx.config.sources.store(Arc::new(sources));
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Success);
let m3u_path = temp.path().join("selected.m3u");
let before_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap();
let before_strm = strm_artifacts(&strm_root).await;
let m3u_keeps_companion = matches!(boundary, LibraryFilterBoundary::Strm);
let strm_keeps_companion = matches!(boundary, LibraryFilterBoundary::M3u);
assert_eq!(before_m3u.matches("#EXTINF").count(), 1 + usize::from(m3u_keeps_companion));
assert_eq!(before_strm.len(), 1 + usize::from(strm_keeps_companion));
assert_eq!(before_m3u.contains("http://stream.example/companion"), m3u_keeps_companion);
assert_eq!(
before_strm.iter().any(|(_, body)| body.contains("http://stream.example/companion")),
strm_keeps_companion
);
ctx.events.0.lock().unwrap().clear();
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
let after_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap();
let after_strm = strm_artifacts(&strm_root).await;
println!("Library filter {boundary:?}, disk={disk}: M3U before={before_m3u:?}, after={after_m3u:?}; STRM before={before_strm:?}, after={after_strm:?}");
assert_run_state(&ctx, PlaylistUpdateState::Success);
assert_eq!(
scan_results(&ctx),
vec![LibraryScanResult {
files_scanned: 0,
groups_scanned: 0,
files_added: 0,
files_updated: 0,
files_removed: 1,
errors: 0
}]
);
assert_ne!(before_m3u, after_m3u, "old Library entries must really be replaced");
assert_eq!(after_m3u.matches("#EXTINF").count(), usize::from(m3u_keeps_companion));
assert_eq!(after_m3u.contains("http://stream.example/companion"), m3u_keeps_companion);
assert_eq!(after_strm.len(), usize::from(strm_keeps_companion));
for (path, contents) in &before_strm {
if contents.contains("http://stream.example/companion") {
assert!(
after_strm.contains(&(path.clone(), contents.clone())),
"valid companion must be retained byte-for-byte"
);
} else {
assert!(!path.exists(), "cleanup must remove old Library artifact {path:?}");
}
}
// The existing M3U database is refreshed too, not only the client text file.
let target_path =
tuliprox_repository::get_target_storage_path(&ctx.config.config.load(), "selected").unwrap();
let mut db = tuliprox_repository::BPlusTreeQuery::<u32, shared::model::M3uPlaylistItem>::try_new(
&tuliprox_repository::m3u_get_file_path_for_db(&target_path),
)
.unwrap();
assert_eq!(db.iter().count(), usize::from(m3u_keeps_companion));
assert!(!temp.path().join("unselected.m3u").exists());
}
}
#[tokio::test]
async fn manual_update_library_processing_filter_replaces_old_output_after_last_movie_removed() {
assert_library_filtered_empty_publication(LibraryFilterBoundary::Processing).await;
}
#[tokio::test]
async fn manual_update_library_persist_filter_replaces_old_output_after_last_movie_removed() {
assert_library_filtered_empty_publication(LibraryFilterBoundary::Persist).await;
}
#[tokio::test]
async fn manual_update_library_m3u_output_filter_replaces_old_output_and_preserves_unfiltered_strm() {
assert_library_filtered_empty_publication(LibraryFilterBoundary::M3u).await;
}
#[tokio::test]
async fn manual_update_library_strm_output_filter_cleans_old_files_and_preserves_unfiltered_m3u() {
assert_library_filtered_empty_publication(LibraryFilterBoundary::Strm).await;
}
#[tokio::test]
async fn manual_update_library_empty_does_not_authorize_foreign_forced_empty_target() {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
let sources = ctx.config.sources.load_full();
let mut playlists = sources
.inputs
.iter()
.map(|input| FetchedPlaylist { input, source: MemoryPlaylistSource::default().into_source(), epg: None })
.collect::<Vec<_>>();
let prepared = prepare_playlist_for_target(
&ctx,
&mut playlists,
&sources.sources[0].targets[0],
&mut HashMap::new(),
&mut Vec::new(),
ClusterFlags::Vod,
false,
)
.await
.unwrap();
assert_eq!(prepared.library_empty, tuliprox_repository::LibraryEmptyPublication::Contribution);
let (result, _) = finalize_prepared_target(Arc::new(ctx), prepared).await;
assert!(result.is_err(), "Library must not relax the existing non-Xtream force-empty output guard");
assert!(!temp.path().join("selected.m3u").exists());
}
#[tokio::test]
async fn manual_update_library_incomplete_discovery_has_no_invented_scan_result_or_rebuild() {
for failure in ["missing", "not-directory", "partial", "no-directories", "corrupt-catalog"] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap();
let status_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await;
let cache_before = input_cache::load_input_status(&status_path).clusters;
ctx.events.0.lock().unwrap().clear();
let mut config = (**ctx.config.config.load()).clone();
let library = config.library.as_mut().unwrap();
match failure {
"missing" => {
library.scan_directories[0].path = temp.path().join("missing").to_string_lossy().into_owned();
}
"not-directory" => {
library.scan_directories[0].path =
temp.path().join("media/Library Movie (2025).mp4").to_string_lossy().into_owned();
}
"partial" => {
let mut missing = library.scan_directories[0].clone();
missing.path = temp.path().join("missing").to_string_lossy().into_owned();
library.scan_directories.push(missing);
}
"no-directories" => library.scan_directories.clear(),
"corrupt-catalog" => {
let path = tuliprox_library::library::resolve_metadata_storage_path(
config.metadata_update.as_ref(),
&config.storage_dir,
)
.join("library/broken.json");
tokio::fs::write(path, "invalid json").await.unwrap();
}
_ => unreachable!(),
}
ctx.config.config.store(Arc::new(config));
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
assert_run_state(&ctx, PlaylistUpdateState::Failure);
assert!(scan_results(&ctx).is_empty(), "{failure}");
let status = input_cache::load_input_status(&status_path);
assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Failure, "{failure}");
assert_eq!(status.clusters, cache_before, "failed rescan must not mutate cache validity");
assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap());
}
}
#[tokio::test]
async fn manual_update_library_selected_target_keeps_companion_and_real_processing_order() {
for (order, expected) in [(ProcessingOrder::Frm, "MAPPED"), (ProcessingOrder::Fmr, "RENAMED")] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), order).await;
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
assert_eq!(output.matches("#EXTINF").count(), 2, "{output}");
assert!(output.contains("http://stream.example/companion"), "{output}");
assert!(output.contains(expected), "{output}");
assert!(!temp.path().join("unselected.m3u").exists());
let events = ctx.events.0.lock().unwrap();
let scan = events
.iter()
.position(|event| {
matches!(event, EventMessage::PlaylistUpdateProgress(progress)
if progress.input_id == Some(2) && progress.message.starts_with("Library rescan completed"))
})
.unwrap();
let acquired = events
.iter()
.position(|event| {
matches!(event, EventMessage::PlaylistUpdateProgress(progress)
if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success))
})
.unwrap();
let published = events
.iter()
.position(|event| {
matches!(event, EventMessage::PlaylistUpdate(summary)
if summary.state == PlaylistUpdateState::Success)
})
.unwrap();
assert!(scan < acquired && acquired < published);
assert!(events.iter().all(|event| !matches!(event, EventMessage::PlaylistUpdateProgress(progress)
if progress.input_id == Some(2) && progress.input_telemetry.as_ref().is_some_and(|telemetry|
!telemetry.clusters.is_empty() || telemetry.refresh_policy != InputRefreshPolicy::NORMAL))));
}
}
#[tokio::test]
async fn manual_update_library_unreadable_catalog_and_busy_scan_do_not_publish() {
for busy in [false, true] {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
let guard = UpdateGuard::new();
let _busy = busy.then(|| guard.try_library().unwrap());
if !busy {
let mut config = (**ctx.config.config.load()).clone();
config.library.as_mut().unwrap().scan_directories[0].path =
temp.path().join("missing").to_string_lossy().into_owned();
ctx.config.config.store(Arc::new(config));
}
exec_processing(rescan_run(&ctx, guard.clone())).await;
assert!(!temp.path().join("selected.m3u").exists());
assert!(!temp.path().join("unselected.m3u").exists());
assert!(ctx
.events
.0
.lock()
.unwrap()
.iter()
.any(|event| matches!(event, EventMessage::PlaylistUpdate(summary)
if summary.state == PlaylistUpdateState::Failure)));
assert!(guard.try_playlist().is_some());
}
}
#[tokio::test]
async fn manual_update_library_input_success_remains_separate_from_target_publish_failure() {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
tokio::fs::create_dir(temp.path().join("selected.m3u")).await.unwrap();
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
let events = ctx.events.0.lock().unwrap();
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress)
if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success))));
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary)
if summary.state == PlaylistUpdateState::Failure)));
}
#[tokio::test]
async fn manual_update_library_rescan_reloads_valid_cache_but_bulk_keeps_catalog() {
let temp = tempfile::tempdir().unwrap();
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
tokio::fs::write(temp.path().join("media/Second Movie (2025).mp4"), b"fixture").await.unwrap();
// Ordinary bulk does not discover the new file and does not acquire a library permit.
let guard = UpdateGuard::new();
let standalone_scan = guard.try_library().unwrap();
exec_processing(
ProcessingRun::new(ctx.client.clone(), ctx.config.clone(), ctx.user_targets.clone(), ctx.events.clone())
.with_update_guard(guard.clone()),
)
.await;
let bulk = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
assert_eq!(bulk.matches("#EXTINF").count(), 2);
drop(standalone_scan);
exec_processing(rescan_run(&ctx, guard)).await;
let rescanned = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
assert_eq!(rescanned.matches("#EXTINF").count(), 3, "{rescanned}");
}
}
+142 -31
View File
@@ -2,7 +2,6 @@ use super::providers::{LibraryProvider, PlexProvider, StalkerProvider, XmltvEpgP
use crate::{
fetched_playlist::FetchedPlaylist,
input_cache,
input_cache::ClusterState,
metadata_sink::{MetadataUpdateSink, NoopMetadataSink},
parser::xmltv::{flatten_tvguide, merge_epg_trees, EpgMergeAccumulator, TVGuide},
playlist_watch::{process_group_watch, process_target_groups_watch},
@@ -24,10 +23,14 @@ use shared::{
error::{get_errors_notify_message, TuliproxError},
foundation::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider},
model::{
ClusterFlags, ConfigTargetOptions, CounterModifier, EventMessage, EventSink, FieldGet, FieldSet, InputStats,
InputType, MappingStage, PipelineStats, PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistStats,
PlaylistUpdateProgressEvent, PlaylistUpdateSummary, ProviderFetchFailure, SourceStats, StreamProperties,
TargetStats, UUIDType, WatchDisabled, WatchDisabledReason, WatchUnmatched, XtreamCluster,
ClusterFlags, ConfigTargetOptions, CounterModifier, EventMessage, EventSink, FieldGet, FieldSet,
InputRefreshOverride, InputRefreshPolicy, InputStats, InputType, InputUpdateAction, InputUpdateRequest,
MappingStage, PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateQualityDecision,
PersistedPlaylistUpdateQualitySnapshot, PersistedPlaylistUpdateTechnicalState, PipelineStats, PlaylistGroup,
PlaylistItem, PlaylistItemType, PlaylistStats, PlaylistUpdateClusterDecision, PlaylistUpdateClusterTelemetry,
PlaylistUpdateDataSource, PlaylistUpdateInputTelemetry, PlaylistUpdateProgressEvent, PlaylistUpdateRunId,
PlaylistUpdateRunOrder, PlaylistUpdateState, PlaylistUpdateSummary, ProviderFetchFailure, SourceStats,
StreamProperties, TargetStats, UUIDType, WatchDisabled, WatchDisabledReason, WatchUnmatched, XtreamCluster,
},
utils::{create_alias_uuid, interner_gc, sanitize_sensitive_info, Internable},
};
@@ -35,7 +38,7 @@ use std::{
collections::{HashMap, HashSet},
future::Future,
path::PathBuf,
sync::{Arc, Weak},
sync::{atomic::AtomicU64, Arc, Weak},
time::{Duration, Instant},
};
use tokio::{
@@ -44,10 +47,10 @@ use tokio::{
};
use tuliprox_core::{
model::{
is_valid, retain_filtered_playlist, AppConfig, CompiledMapping, ConfigFavourites, ConfigInput,
ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, Epg, FilterOutcome, MappingProgram,
ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig, TransformStage, UpdateGuard,
UpdateTask,
is_valid, retain_filtered_playlist, AppConfig, ClusterForceUpdate, ClusterUpdateRejection, CompiledMapping,
ConfigFavourites, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, Epg,
FilterOutcome, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig,
TransformStage, UpdateGuard, UpdateTask,
},
utils::{debug_if_enabled, log_memory_snapshot, trace_if_enabled, StepMeasure, StepMeasureCallback},
};
@@ -62,15 +65,28 @@ use tuliprox_iptv::{
xtream,
};
use tuliprox_repository::{
load_input_playlist, persist_input_playlist, persist_playlist, CategoryKey, MemoryPlaylistSource, PlaylistSource,
PlaylistStorageState,
load_input_playlist, persist_input_playlist_with_options, persist_playlist, CategoryKey,
InputPlaylistPersistOptions, MemoryPlaylistSource, PlaylistSource, PlaylistStorageState,
TargetPlaylistPersistOptions,
};
use tuliprox_session::ActiveProviderManager;
const PLAYLIST_UPDATE_MAX_DURATION_SECS: u64 = 3600;
const MAX_CONCURRENT_TARGET_FINALIZERS: usize = 2;
static NEXT_PLAYLIST_UPDATE_RUN_ORDER: AtomicU64 = AtomicU64::new(1);
/// Allocates the order at which a run entered the serial execution boundary.
#[must_use]
pub fn next_playlist_update_run_order() -> PlaylistUpdateRunOrder {
PlaylistUpdateRunOrder::from(NEXT_PLAYLIST_UPDATE_RUN_ORDER.fetch_add(1, std::sync::atomic::Ordering::SeqCst))
}
mod fetch_outcome;
mod ingest;
mod input_status;
mod m3u_quality;
mod manual_update;
use manual_update::LibraryUpdateMode;
mod target;
mod transform;
@@ -123,6 +139,7 @@ pub struct ProcessingRun<
M: MetadataUpdateSink = NoopMetadataSink,
> {
client: reqwest::Client,
run_id: PlaylistUpdateRunId,
app_config: Arc<AppConfig>,
targets: Arc<ProcessTargets>,
events: E,
@@ -134,12 +151,27 @@ pub struct ProcessingRun<
metadata_manager: Option<Arc<M>>,
pre_processed_inputs: Option<HashSet<Arc<str>>>,
acquired_permit: Option<tuliprox_core::model::UpdateGuardPermit>,
input_refresh: Option<InputRefreshOverride>,
library_update_mode: LibraryUpdateMode,
}
impl<E: EventSink + Clone + 'static> ProcessingRun<E, NoBootstrap, NoopMetadataSink> {
pub fn new(client: reqwest::Client, app_config: Arc<AppConfig>, targets: Arc<ProcessTargets>, events: E) -> Self {
Self::for_run(PlaylistUpdateRunId::generate(), client, app_config, targets, events)
}
/// Creates a run for an identity already allocated by an accepted queue request.
#[must_use]
pub fn for_run(
run_id: PlaylistUpdateRunId,
client: reqwest::Client,
app_config: Arc<AppConfig>,
targets: Arc<ProcessTargets>,
events: E,
) -> Self {
Self {
client,
run_id,
app_config,
targets,
events,
@@ -151,6 +183,8 @@ impl<E: EventSink + Clone + 'static> ProcessingRun<E, NoBootstrap, NoopMetadataS
metadata_manager: None,
pre_processed_inputs: None,
acquired_permit: None,
input_refresh: None,
library_update_mode: LibraryUpdateMode::ExistingCatalog,
}
}
}
@@ -164,6 +198,7 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
pub fn with_bootstrap<B2: UpdateBootstrap>(self, bootstrap: B2) -> ProcessingRun<E, B2, M> {
ProcessingRun {
client: self.client,
run_id: self.run_id,
app_config: self.app_config,
targets: self.targets,
events: self.events,
@@ -175,6 +210,8 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
metadata_manager: self.metadata_manager,
pre_processed_inputs: self.pre_processed_inputs,
acquired_permit: self.acquired_permit,
input_refresh: self.input_refresh,
library_update_mode: self.library_update_mode,
}
}
@@ -211,6 +248,7 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
pub fn with_metadata_manager<M2: MetadataUpdateSink>(self, manager: Arc<M2>) -> ProcessingRun<E, B, M2> {
ProcessingRun {
client: self.client,
run_id: self.run_id,
app_config: self.app_config,
targets: self.targets,
events: self.events,
@@ -222,6 +260,8 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
metadata_manager: Some(manager),
pre_processed_inputs: self.pre_processed_inputs,
acquired_permit: self.acquired_permit,
input_refresh: self.input_refresh,
library_update_mode: self.library_update_mode,
}
}
@@ -240,6 +280,30 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
self.acquired_permit = permit.into();
self
}
/// Carries a validated card action through the existing serial run.
#[must_use]
pub fn with_manual_input_update(mut self, request: Option<InputUpdateRequest>) -> Self {
if let Some(request) = request {
match request.action {
InputUpdateAction::Provider(policy) => {
self.input_refresh = Some(InputRefreshOverride { input_id: request.input_id, policy });
}
InputUpdateAction::Rescan => {
self.library_update_mode = LibraryUpdateMode::Rescan { input_id: request.input_id };
self.input_refresh = None;
}
}
}
self
}
/// Applies one manual policy override without changing the input configuration.
#[must_use]
pub fn with_input_refresh(mut self, input_refresh: impl Into<Option<InputRefreshOverride>>) -> Self {
self.input_refresh = input_refresh.into();
self
}
}
#[allow(clippy::too_many_lines)]
@@ -248,6 +312,7 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
) {
let ProcessingRun {
client,
run_id,
app_config,
targets,
events,
@@ -259,6 +324,8 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
metadata_manager,
pre_processed_inputs,
acquired_permit,
input_refresh,
library_update_mode,
} = run;
let max_update_duration = Duration::from_secs(PLAYLIST_UPDATE_MAX_DURATION_SECS);
@@ -269,7 +336,10 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
Some(permit)
} else {
warn!("Playlist update lock is closed; update skipped.");
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
let execution_order = next_playlist_update_run_order();
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
run_id,
execution_order,
shared::model::PlaylistUpdateState::Failure,
)));
return;
@@ -277,6 +347,7 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
} else {
None
};
let execution_order = next_playlist_update_run_order();
if playlist_guard.is_some() {
if let Some(bootstrap) = bootstrap.as_ref() {
@@ -284,7 +355,9 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
error!(
"Playlist update bootstrap timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
);
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
run_id,
execution_order,
shared::model::PlaylistUpdateState::Failure,
)));
return;
@@ -306,11 +379,14 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
// Initialize Context
let ctx = PlaylistProcessingContext {
client,
run_id: run_id.clone(),
execution_order,
config: app_config.clone(),
user_targets: targets.clone(),
events: events.clone(),
playlist_state: playlist_state.clone(),
processed_inputs: Arc::new(Mutex::new(HashSet::new())),
input_completions: Arc::new(Mutex::new(HashMap::new())),
input_locks: Arc::new(Mutex::new(HashMap::new())),
disabled_headers,
provider_manager,
@@ -324,17 +400,24 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
StalkerRefreshMode::Complete
},
partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)),
had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)),
input_refresh,
library_update_mode,
};
let start_time = Instant::now();
let process_result =
tokio::time::timeout(max_update_duration, std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind())
.await;
let process_result = tokio::time::timeout(
max_update_duration,
std::panic::AssertUnwindSafe(manual_update::process_manual_update(&ctx, update_guard.as_ref())).catch_unwind(),
)
.await;
let (stats, errors) = match process_result {
Ok(Ok((stats, errors))) => (stats, errors),
Ok(Err(_)) => {
error!("Playlist processing panicked");
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
run_id,
execution_order,
shared::model::PlaylistUpdateState::Failure,
)));
return;
@@ -343,7 +426,9 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
error!(
"Playlist processing timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
);
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
run_id,
execution_order,
shared::model::PlaylistUpdateState::Failure,
)));
return;
@@ -373,22 +458,29 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
// refresh notified twice. Subscribers now get one event with everything,
// and the bridge renders the single message from it.
let error = get_errors_notify_message!(errors, 255);
let outcome = if error.is_some() {
shared::model::PlaylistUpdateState::Failure
} else if ctx.partial_refresh.load(std::sync::atomic::Ordering::Acquire) {
shared::model::PlaylistUpdateState::Partial
} else {
shared::model::PlaylistUpdateState::Success
};
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary { state: outcome, stats, error }));
let outcome = PlaylistRunSignals {
has_error: error.is_some(),
has_pending_stalker_refresh: ctx.partial_refresh.load(std::sync::atomic::Ordering::Acquire),
has_quality_rejections: ctx.had_quality_rejections.load(std::sync::atomic::Ordering::Acquire),
}
.state();
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary {
run_id: Some(run_id.clone()),
execution_order: Some(execution_order),
state: outcome,
stats,
error,
}));
let elapsed = start_time.elapsed().as_secs();
let update_finished_message = format!("🌷 Update process finished! Took {elapsed} secs.");
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent {
target: "Playlist Update".to_string(),
message: update_finished_message.clone(),
}));
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_global(
run_id,
execution_order,
"Playlist Update",
update_finished_message.clone(),
)));
log_memory_snapshot("exec_processing before_interner_gc");
debug!("StringInterner GC removed {} strings", interner_gc());
log_memory_snapshot("exec_processing after_interner_gc");
@@ -397,5 +489,24 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
info!("{update_finished_message}");
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(crate) struct PlaylistRunSignals {
pub(crate) has_error: bool,
pub(crate) has_pending_stalker_refresh: bool,
pub(crate) has_quality_rejections: bool,
}
impl PlaylistRunSignals {
pub(crate) const fn state(self) -> PlaylistUpdateState {
if self.has_error {
PlaylistUpdateState::Failure
} else if self.has_pending_stalker_refresh || self.has_quality_rejections {
PlaylistUpdateState::Partial
} else {
PlaylistUpdateState::Success
}
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,166 @@
mod staged_completion {
use super::*;
pub(super) fn input_completions(events: &CollectSink, input_id: u16) -> Vec<PlaylistUpdateProgressEvent> {
events
.0
.lock()
.unwrap()
.iter()
.filter_map(|event| match event {
EventMessage::PlaylistUpdateProgress(progress)
if progress.input_id == Some(input_id) && progress.state.is_some() =>
{
Some(progress.clone())
}
_ => None,
})
.collect()
}
async fn staged_run_context(
root: &Path,
base_url: &str,
) -> (PlaylistProcessingContext<CollectSink>, Arc<ConfigInput>) {
let ctx = processing_context_with_events(root, CollectSink::default());
let parent_file = root.join("parent.m3u");
std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap();
let parent = Arc::new(ConfigInput {
id: 7,
name: "parent".intern(),
input_type: InputType::M3u,
url: parent_file.to_string_lossy().into_owned(),
enabled: true,
..ConfigInput::default()
});
let mut staged = (*test_input(
base_url,
ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 },
0,
&[XtreamCluster::Video, XtreamCluster::Series],
))
.clone();
staged.id = 8;
staged.name = "staged".intern();
staged.input_type = InputType::Staged;
staged.staged_type = shared::model::StagedInputType::Xtream;
staged.staged = Some(tuliprox_core::model::ConfigInputStaged {
for_input: Some(parent.name.clone()),
clusters: ClusterFlags::Live,
});
staged.resolve_staged_download_type();
let staged = Arc::new(staged);
seed_live_baseline(&ctx, &staged, 100).await;
let failed_companion = Arc::new(ConfigInput {
id: 9,
name: "failed-companion".intern(),
input_type: InputType::M3u,
url: root.join("missing.m3u").to_string_lossy().into_owned(),
enabled: true,
..ConfigInput::default()
});
ctx.config.sources.store(Arc::new(SourcesConfig {
inputs: vec![parent.clone(), staged.clone(), failed_companion.clone()],
sources: vec![ConfigSource {
inputs: vec![parent.name.clone(), failed_companion.name.clone()],
targets: vec![xtream_target(true)],
}],
..SourcesConfig::default()
}));
(ctx, staged)
}
fn normalized_trace(
events: &[EventMessage],
persisted: shared::model::PersistedPlaylistUpdateInputResult,
) -> serde_json::Value {
let summary = events
.iter()
.find_map(|event| match event {
EventMessage::PlaylistUpdate(summary) => Some(summary),
_ => None,
})
.unwrap();
assert_eq!(summary.run_id, Some("staged-completion-run".into()));
assert!(summary.execution_order.is_some());
assert_eq!(summary.state, PlaylistUpdateState::Failure);
let progress: Vec<_> = events
.iter()
.filter_map(|event| match event {
EventMessage::PlaylistUpdateProgress(progress)
if progress.input_id == Some(8) || progress.state.is_some() =>
{
assert_eq!(progress.run_id, summary.run_id);
assert_eq!(progress.execution_order, summary.execution_order);
let mut progress = progress.clone();
// Only the globally allocated test-run order is normalized. All
// input identities, states, messages and telemetry remain exact.
progress.execution_order = Some(1.into());
Some(progress)
}
_ => None,
})
.collect();
assert!(persisted.timestamp > 0);
serde_json::json!({
"progress": progress,
"completed": {"run_id": summary.run_id, "execution_order": 1, "state": summary.state},
"persisted_input_result": {"state": persisted.state, "timestamp": 1}
})
}
#[tokio::test]
async fn playlist_update_status_staged_reporter_trace_matches_persistence_before_foreign_run_failure() {
let mut traces = Vec::new();
for expected in [PlaylistUpdateState::Partial, PlaylistUpdateState::Success, PlaylistUpdateState::Failure] {
let temp = tempfile::tempdir().unwrap();
let mut responses =
fixture_responses([if expected == PlaylistUpdateState::Success { 100 } else { 1 }, 0, 0]);
if expected == PlaylistUpdateState::Failure {
responses.insert("get_live_streams".into(), "not JSON".into());
}
let server = TestXtreamServer::start_with_responses(responses);
let (ctx, staged) = staged_run_context(temp.path(), &server.base_url).await;
exec_processing(ProcessingRun::for_run(
"staged-completion-run".into(),
ctx.client.clone(),
ctx.config.clone(),
ctx.user_targets.clone(),
ctx.events.clone(),
))
.await;
let path = input_storage_path(&ctx, &staged).await;
let persisted = input_cache::load_input_status(&path);
assert!(!persisted.clusters.contains_key("default"));
assert_eq!(persisted.last_input_update.unwrap().state, expected);
let completions = input_completions(&ctx.events, staged.id);
assert_eq!(completions.len(), 1, "real indirect staged input must emit one correlated completion");
assert_eq!(completions[0].state, Some(expected));
let telemetry = completions[0].input_telemetry.as_ref().unwrap();
assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL);
assert_eq!(telemetry.source, None);
assert_eq!(telemetry.clusters[0].source, Some(PlaylistUpdateDataSource::Provider));
assert_eq!(telemetry.clusters[0].threshold, Some(95));
if expected == PlaylistUpdateState::Partial {
assert_eq!(telemetry.clusters[0].decision, Some(PlaylistUpdateClusterDecision::Rejected));
assert_eq!(telemetry.clusters[0].active_count, Some(100));
}
let emitted = ctx.events.0.lock().unwrap().clone();
let own_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(8) && p.state.is_some())).unwrap();
let parent_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(7) && p.state.is_some())).unwrap();
let completed_index =
emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdate(_))).unwrap();
assert!(own_index < parent_index && parent_index < completed_index);
assert_eq!(input_completions(&ctx.events, 9)[0].state, Some(PlaylistUpdateState::Failure));
traces.push(normalized_trace(&emitted, persisted.last_input_update.unwrap()));
assert_eq!(server.finish().iter().filter(|action| action.as_str() == "get_live_streams").count(), 1);
}
let actual = serde_json::Value::Array(traces);
let fixture: serde_json::Value = serde_json::from_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../frontend/src/model/test_data/staged_completions.json"
)))
.unwrap();
assert_eq!(actual, fixture, "actual reporter trace: {}", serde_json::to_string_pretty(&actual).unwrap());
}
}
@@ -93,14 +93,23 @@ pub(crate) async fn process_targets<E: EventSink + Clone + 'static, M: MetadataU
targets: &[&Arc<ConfigTarget>],
input_stats: &mut HashMap<Arc<str>, InputStats>,
errors: &mut Vec<TuliproxError>,
accepted_empty_clusters: ClusterFlags,
process_parallel: bool,
) -> Vec<TargetStats> {
if !process_parallel {
let mut target_stats = Vec::with_capacity(targets.len());
for (index, target) in targets.iter().enumerate() {
let consume_input_source = index + 1 == targets.len();
let result =
prepare_playlist_for_target(ctx, playlists, target, input_stats, errors, consume_input_source).await;
let result = prepare_playlist_for_target(
ctx,
playlists,
target,
input_stats,
errors,
accepted_empty_clusters,
consume_input_source,
)
.await;
match result {
Ok(prepared) => {
let processing = prepared.processing.clone();
@@ -142,7 +151,9 @@ pub(crate) async fn process_targets<E: EventSink + Clone + 'static, M: MetadataU
let (completion, receiver) = watch::channel(false);
completion_receivers.push(receiver);
match prepare_playlist_for_target(ctx, playlists, target, input_stats, errors, false).await {
match prepare_playlist_for_target(ctx, playlists, target, input_stats, errors, accepted_empty_clusters, false)
.await
{
Ok(prepared) => {
let processing = prepared.processing.clone();
let task_ctx = Arc::clone(ctx);
@@ -232,10 +243,12 @@ pub(crate) fn apply_persist_filter(target: &ConfigTarget, groups: &mut Vec<Playl
}
pub(crate) struct PreparedTarget {
pub(crate) library_empty: tuliprox_repository::LibraryEmptyPublication,
pub(crate) target: ConfigTarget,
pub(crate) playlist: Vec<PlaylistGroup>,
pub(crate) epg: Vec<Epg>,
pub(crate) processing: PipelineStats,
pub(crate) accepted_empty_clusters: ClusterFlags,
}
#[allow(clippy::too_many_arguments)]
@@ -245,6 +258,7 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
target: &ConfigTarget,
stats: &mut HashMap<Arc<str>, InputStats>,
errors: &mut Vec<TuliproxError>,
accepted_empty_clusters: ClusterFlags,
consume_input_source: bool,
) -> Result<PreparedTarget, Vec<TuliproxError>> {
debug_if_enabled!("Processing order is {}", &target.processing_order);
@@ -254,9 +268,15 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
let mut new_epg = vec![];
let mut new_playlist: Vec<PlaylistGroup> = vec![];
let mut aggregate_outcome = PipelineOutcome::default();
let mut has_empty_library = false;
let mut all_inputs_empty = !playlists.is_empty();
let mut only_library_inputs = true;
// The caller has already required Ready for every input. An empty foreign input
// still needs its own Force contract; it cannot borrow Library's filter permission.
let mut non_library_inputs_populated = true;
debug!("Executing processing pipes");
let broadcast_step = create_broadcast_callback(&ctx.events);
let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order);
let bouquet_file =
tuliprox_repository::load_target_bouquet(&ctx.config, &target.name).await.map_err(|err| vec![err])?;
@@ -270,6 +290,11 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
let pipe = get_processing_pipe(target);
let mut step = StepMeasure::new(&target.name, broadcast_step);
for provider_fpl in playlists.iter_mut() {
let input_is_empty = provider_fpl.source.is_empty();
all_inputs_empty &= input_is_empty;
only_library_inputs &= provider_fpl.input.input_type == InputType::Library;
non_library_inputs_populated &= provider_fpl.input.input_type == InputType::Library || !input_is_empty;
has_empty_library |= provider_fpl.input.input_type == InputType::Library && input_is_empty;
log_memory_snapshot(
format!("target '{}' input '{}' before_pipe", target.name, provider_fpl.input.name).as_str(),
);
@@ -319,10 +344,19 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
log_memory_snapshot(format!("target '{}' after_filter_rename_map_epg", target.name).as_str());
step.stop("Preparing playlist");
Ok(PreparedTarget {
library_empty: match (has_empty_library, all_inputs_empty && only_library_inputs) {
(true, true) => tuliprox_repository::LibraryEmptyPublication::CompleteTarget,
(true, false) if non_library_inputs_populated && accepted_empty_clusters.is_empty() => {
tuliprox_repository::LibraryEmptyPublication::FilterableContribution
}
(true, false) => tuliprox_repository::LibraryEmptyPublication::Contribution,
(false, _) => tuliprox_repository::LibraryEmptyPublication::None,
},
target: target.clone(),
playlist: new_playlist,
epg: new_epg,
processing: aggregate_outcome.to_stats(),
accepted_empty_clusters,
})
}
@@ -373,7 +407,7 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
let mut new_playlist = prepared.playlist;
let mut new_epg = prepared.epg;
let mut errors = Vec::new();
let broadcast_step = create_broadcast_callback(&ctx.events);
let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order);
let mut step = StepMeasure::new(&target.name, broadcast_step);
if target.favourites.is_some() {
step.broadcast("Processing favourites for '{}' playlist", &target.name);
@@ -381,7 +415,9 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
log_memory_snapshot(format!("target '{}' after_favourites", target.name).as_str());
}
if new_playlist.is_empty() {
let library_empty = prepared.library_empty.for_filtered_playlist(&new_playlist);
if new_playlist.is_empty() && prepared.accepted_empty_clusters.is_empty() && !library_empty.replaces_empty_target()
{
step.stop("");
info!("Playlist is empty: {}", target.name);
(Ok(()), errors)
@@ -455,12 +491,14 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
} else {
flatten_tvguide(new_epg)
};
let library_empty = library_empty.for_filtered_playlist(&flat_new_playlist);
let result = persist_playlist(
&ctx.config,
&mut flat_new_playlist,
merged_epg.as_ref(),
target,
ctx.playlist_state.as_ref(),
TargetPlaylistPersistOptions { accepted_empty_clusters: prepared.accepted_empty_clusters, library_empty },
)
.await;
if result.is_ok() && process_watch(&ctx.config, &ctx.events, target, &flat_new_playlist).await {
File diff suppressed because it is too large Load Diff
@@ -41,16 +41,16 @@ impl PlaylistProvider for StalkerProvider {
fn name(&self) -> &'static str { "stalker" }
async fn fetch(&self, request: &PlaylistFetchRequest<'_>) -> PlaylistFetch {
let (groups, errors, persisted, partial) = download_stalker_playlist(
download_stalker_playlist(
request.app_config,
request.client,
request.input,
None,
self.refresh_mode,
self.materialize_active,
request.update_quality,
)
.await;
PlaylistFetch::groups(groups).with_errors(errors).persisted(persisted).partial(partial)
.await
}
}
+119 -45
View File
@@ -10,15 +10,15 @@
#![allow(clippy::too_many_lines, clippy::needless_pass_by_value)]
use super::stalker_refresh::{
advance_stalker_refresh, StalkerClusterSelection, StalkerRefreshMode, StalkerRefreshOutcome,
};
use super::stalker_refresh::{advance_stalker_refresh, StalkerRefreshMode, StalkerRefreshOutcome, StalkerRefreshPlan};
use log::{debug, info, warn};
use lru::LruCache;
use parking_lot::Mutex;
use shared::{
error::TuliproxError,
model::{stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, PlaylistGroup, PlaylistItem},
model::{
stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, PlaylistGroup, PlaylistItem, UpdateQualityPolicy,
},
utils::Internable,
};
use std::{
@@ -29,7 +29,10 @@ use std::{
time::{Duration, Instant},
};
use tuliprox_core::model::{AppConfig, ConfigInput, ConfigInputFlags, StalkerInputConfig};
use tuliprox_iptv::stalker::{client::StalkerApiClient, error::StalkerError, parser};
use tuliprox_iptv::{
provider::PlaylistFetch,
stalker::{client::StalkerApiClient, error::StalkerError, parser},
};
use tuliprox_repository::{
stalker_generation_repository::{load_active_manifest, load_checkpoint},
stalker_repository::{ensure_stalker_storage_path, load_stalker_items_at, read_stalker_item_at},
@@ -87,6 +90,24 @@ async fn try_acquire_stalker_refresh(input_name: &Arc<str>) -> Option<tokio::syn
semaphore.try_acquire_owned().ok()
}
fn stalker_refresh_busy_fetch(input_name: &str, persisted: bool, quality_policy: UpdateQualityPolicy) -> PlaylistFetch {
if matches!(quality_policy, UpdateQualityPolicy::Bypass) {
PlaylistFetch::failed(TuliproxError::ProviderConnection(format!(
"Stalker refresh for input '{input_name}' is busy; retry the forced update"
)))
.persisted(persisted)
} else {
PlaylistFetch::nothing_to_do().persisted(persisted).partial(true)
}
}
fn failed_stalker_acquisition(error: TuliproxError, clusters: &[StalkerCluster]) -> PlaylistFetch {
PlaylistFetch {
failed_clusters: clusters.iter().copied().map(xtream_cluster).collect(),
..PlaylistFetch::failed(error)
}
}
fn cached_resolved_link(key: RuntimeLinkKey, force_refresh: bool) -> Option<Arc<str>> {
let mut cache = RUNTIME_STALKER_LINKS.lock();
let expired = cache.peek(&key).is_some_and(|entry| entry.expires_at <= Instant::now());
@@ -111,19 +132,15 @@ pub async fn download_stalker_playlist(
clusters: Option<&[StalkerCluster]>,
refresh_mode: StalkerRefreshMode,
materialize_active: bool,
) -> (Vec<PlaylistGroup>, Vec<TuliproxError>, bool, bool) {
quality_policy: UpdateQualityPolicy,
) -> PlaylistFetch {
let stalker_cfg = match input.stalker.as_ref() {
Some(cfg) => cfg.clone(),
None => {
return (
vec![],
vec![TuliproxError::ConfigInput(format!(
"Stalker input '{}' has no stalker configuration block",
input.name
))],
false,
false,
);
return PlaylistFetch::failed(TuliproxError::ConfigInput(format!(
"Stalker input '{}' has no stalker configuration block",
input.name
)));
}
};
@@ -136,27 +153,22 @@ pub async fn download_stalker_playlist(
if resolved_clusters.is_empty() {
info!("Stalker input '{}' has all clusters skipped", input.name);
return (vec![], vec![], false, false);
return PlaylistFetch::nothing_to_do();
}
let refresh_selection = StalkerClusterSelection::requested(input, &resolved_clusters);
let refresh_plan = StalkerRefreshPlan::requested(input, &resolved_clusters, quality_policy);
let portal_url = match resolve_stalker_portal_url(input) {
Ok(url) => url,
Err(err) => return (vec![], vec![err], false, false),
Err(err) => return failed_stalker_acquisition(err, &resolved_clusters),
};
let identity_fingerprint = stalker_identity_fingerprint(&portal_url, &stalker_cfg);
let api_client = match cached_runtime_stalker_client(client, portal_url, &stalker_cfg) {
Ok(client) => client,
Err(err) => {
return (
vec![],
vec![TuliproxError::ConfigInput(format!(
"failed to build Stalker client for input '{}': {err}",
input.name
))],
false,
false,
return failed_stalker_acquisition(
TuliproxError::ConfigInput(format!("failed to build Stalker client for input '{}': {err}", input.name)),
&resolved_clusters,
);
}
};
@@ -164,11 +176,9 @@ pub async fn download_stalker_playlist(
let storage_path = match ensure_stalker_storage_path(app_config, &input.name).await {
Ok(p) => p,
Err(err) => {
return (
vec![],
vec![TuliproxError::Io(format!("could not prepare Stalker storage for input '{}': {err}", input.name))],
false,
false,
return failed_stalker_acquisition(
TuliproxError::Io(format!("could not prepare Stalker storage for input '{}': {err}", input.name)),
&resolved_clusters,
);
}
};
@@ -178,14 +188,12 @@ pub async fn download_stalker_playlist(
let handshake = match api_client.handshake().await {
Ok(handshake) => handshake,
Err(err) => {
return (
vec![],
vec![TuliproxError::ProviderConnection(format!(
return failed_stalker_acquisition(
TuliproxError::ProviderConnection(format!(
"Stalker handshake for input '{}' failed: {err}",
input.name
))],
false,
false,
)),
&resolved_clusters,
);
}
};
@@ -194,7 +202,7 @@ pub async fn download_stalker_playlist(
app_config,
api_client.as_ref(),
&handshake,
refresh_selection,
refresh_plan,
&storage_path,
identity_fingerprint,
refresh_mode.budget(),
@@ -205,7 +213,7 @@ pub async fn download_stalker_playlist(
} else {
let checkpoint = match load_checkpoint(&storage_path, identity_fingerprint).await {
Ok(checkpoint) => checkpoint,
Err(err) => return (Vec::new(), vec![err], false, false),
Err(err) => return PlaylistFetch::failed(err),
};
break StalkerRefreshOutcome::Yielded {
phase: checkpoint.as_ref().map_or(
@@ -227,18 +235,40 @@ pub async fn download_stalker_playlist(
tokio::task::yield_now().await;
}
Ok(outcome) => break outcome,
Err(err) => return (Vec::new(), vec![err], false, false),
Err(err) => return PlaylistFetch::failed(err),
}
}
} else {
return (Vec::new(), Vec::new(), app_config.config.load().disk_based_processing, true);
return stalker_refresh_busy_fetch(&input.name, app_config.config.load().disk_based_processing, quality_policy);
};
let yielded = matches!(&outcome, StalkerRefreshOutcome::Yielded { .. });
let terminal = matches!(&outcome, StalkerRefreshOutcome::Terminal(_));
let mut errors = Vec::new();
let mut quality_acceptances = Vec::new();
let mut quality_rejections = Vec::new();
let mut force_updates = Vec::new();
match outcome {
StalkerRefreshOutcome::Complete => {}
StalkerRefreshOutcome::Complete {
quality_acceptances: completed_acceptances,
quality_rejections: completed_rejections,
force_updates: completed_force_updates,
} => {
quality_acceptances = completed_acceptances;
quality_rejections = completed_rejections;
force_updates = completed_force_updates;
}
StalkerRefreshOutcome::Failed {
quality_acceptances: completed_acceptances,
quality_rejections: completed_rejections,
force_updates: completed_force_updates,
error,
} => {
return PlaylistFetch::failed(error)
.with_quality_acceptances(completed_acceptances)
.with_quality_rejections(completed_rejections)
.with_force_updates(completed_force_updates);
}
StalkerRefreshOutcome::Yielded { phase, processed, skipped, error } => {
info!(
"Stalker input '{}' yielded in phase {phase:?} after {processed} records ({skipped} skipped)",
@@ -255,7 +285,12 @@ pub async fn download_stalker_playlist(
Ok(manifest) => manifest,
Err(err) => {
errors.push(err);
return (Vec::new(), errors, false, yielded);
return PlaylistFetch::groups(Vec::new())
.with_errors(errors)
.with_quality_acceptances(quality_acceptances)
.with_quality_rejections(quality_rejections)
.with_force_updates(force_updates)
.partial(yielded);
}
};
if terminal {
@@ -268,7 +303,13 @@ pub async fn download_stalker_playlist(
}
let use_disk_based_processing = app_config.config.load().disk_based_processing;
if !materialize_active {
return (Vec::new(), errors, use_disk_based_processing, yielded);
return PlaylistFetch::groups(Vec::new())
.with_errors(errors)
.with_quality_acceptances(quality_acceptances)
.with_quality_rejections(quality_rejections)
.with_force_updates(force_updates)
.persisted(use_disk_based_processing)
.partial(yielded);
}
let mut groups = Vec::new();
let mut counts = [0_usize; 3];
@@ -324,7 +365,13 @@ pub async fn download_stalker_playlist(
}
}
parser::log_stalker_download_summary(&input.name, counts[0], counts[1], counts[2]);
(groups, errors, use_disk_based_processing, yielded)
PlaylistFetch::groups(groups)
.with_errors(errors)
.with_quality_acceptances(quality_acceptances)
.with_quality_rejections(quality_rejections)
.with_force_updates(force_updates)
.persisted(use_disk_based_processing)
.partial(yielded)
}
/// Resolve a per-input portal URL using the standard `provider://` resolution pipeline.
@@ -584,4 +631,31 @@ mod tests {
drop(first);
assert!(try_acquire_stalker_refresh(&name).await.is_some());
}
#[tokio::test]
async fn force_stalker_refresh_busy_is_visible_and_does_not_mutate_checkpoint() {
use tuliprox_repository::stalker_generation_repository::{save_checkpoint, StalkerCheckpoint};
let temp = tempfile::tempdir().expect("temporary Stalker storage");
let input_name: Arc<str> = "force-busy-portal".into();
let identity = 17;
save_checkpoint(temp.path(), &StalkerCheckpoint::new(identity, 23, 0b1111, 123))
.await
.expect("checkpoint should persist");
let refresh_permit = try_acquire_stalker_refresh(&input_name).await.expect("first refresh owns permit");
let fetch = stalker_refresh_busy_fetch(&input_name, true, UpdateQualityPolicy::Bypass);
assert_eq!(fetch.errors.len(), 1);
assert!(!fetch.partial);
assert!(fetch.persisted);
let checkpoint = load_checkpoint(temp.path(), identity)
.await
.expect("checkpoint should load")
.expect("checkpoint should remain present");
assert_eq!(checkpoint.quality_bypass_mask, 0);
drop(refresh_permit);
assert!(try_acquire_stalker_refresh(&input_name).await.is_some());
}
}
@@ -1,7 +1,7 @@
use super::stalker::StalkerCluster;
use shared::{
error::TuliproxError,
model::{stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem},
model::{stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, UpdateQualityPolicy, XtreamCluster},
};
use std::{
collections::HashMap,
@@ -9,7 +9,10 @@ use std::{
sync::Arc,
time::{Instant, SystemTime, UNIX_EPOCH},
};
use tuliprox_core::model::{AppConfig, ConfigInput, ConfigInputFlags};
use tuliprox_core::model::{
evaluate_update_quality, AppConfig, ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection,
ConfigInput, ConfigInputFlags, ConfigInputUpdateQuality,
};
use tuliprox_iptv::stalker::{
catalog::{StalkerCategory, StalkerRawItem},
client::StalkerApiClient,
@@ -23,13 +26,19 @@ use tuliprox_repository::{
publish_selection, save_checkpoint, StalkerCheckpoint, StalkerGenerationData, StalkerRefreshPhase,
},
stalker_repository::{
load_stalker_items_after, prepare_stalker_episode_series_at, promote_stalker_file, remove_stalker_file,
snapshot_stalker_epg_at, snapshot_stalker_items_at, upsert_stalker_epg_at, upsert_stalker_items_at,
count_stalker_items_at, load_stalker_items_after, prepare_stalker_episode_series_at, promote_stalker_file,
remove_stalker_file, snapshot_stalker_epg_at, snapshot_stalker_items_at, upsert_stalker_epg_at,
upsert_stalker_items_at,
},
};
const MAX_RETRIES: u8 = 3;
const SKIPPED_SAMPLE_LIMIT: usize = 32;
const LIVE_SELECTION: u8 = 0b0001;
const VOD_SELECTION: u8 = 0b0010;
const SERIES_SELECTION: u8 = 0b0100;
const EPG_SELECTION: u8 = 0b1000;
const MEDIA_SELECTION: u8 = LIVE_SELECTION | VOD_SELECTION | SERIES_SELECTION;
pub enum StalkerRefreshLimit {
Deadline(Instant),
@@ -118,6 +127,26 @@ impl StalkerClusterSelection {
}
}
/// Requested clusters and the publication policy applied once their generation is complete.
#[derive(Clone, Copy)]
pub struct StalkerRefreshPlan {
selection: StalkerClusterSelection,
update_quality: ConfigInputUpdateQuality,
quality_bypass_mask: u8,
}
impl StalkerRefreshPlan {
pub fn requested(input: &ConfigInput, requested: &[StalkerCluster], quality_policy: UpdateQualityPolicy) -> Self {
let selection = StalkerClusterSelection::requested(input, requested);
let update_quality =
input.options.as_ref().map_or_else(ConfigInputUpdateQuality::default, |options| options.update_quality);
let quality_bypass_mask = matches!(quality_policy, UpdateQualityPolicy::Bypass)
.then_some(selection.mask() & MEDIA_SELECTION)
.unwrap_or(0);
Self { selection, update_quality, quality_bypass_mask }
}
}
fn first_phase(selection: StalkerClusterSelection) -> StalkerRefreshPhase {
if selection.live {
StalkerRefreshPhase::LiveBulk
@@ -154,8 +183,23 @@ fn next_phase_after_series(selection: StalkerClusterSelection) -> StalkerRefresh
#[derive(Debug)]
pub enum StalkerRefreshOutcome {
Complete,
Yielded { phase: StalkerRefreshPhase, processed: u64, skipped: u64, error: Option<TuliproxError> },
Complete {
quality_acceptances: Vec<ClusterUpdateAcceptance>,
quality_rejections: Vec<ClusterUpdateRejection>,
force_updates: Vec<ClusterForceUpdate>,
},
Failed {
quality_acceptances: Vec<ClusterUpdateAcceptance>,
quality_rejections: Vec<ClusterUpdateRejection>,
force_updates: Vec<ClusterForceUpdate>,
error: TuliproxError,
},
Yielded {
phase: StalkerRefreshPhase,
processed: u64,
skipped: u64,
error: Option<TuliproxError>,
},
Terminal(TuliproxError),
}
@@ -169,9 +213,15 @@ async fn load_or_start_checkpoint(
storage_path: &Path,
identity_fingerprint: u64,
selection: StalkerClusterSelection,
quality_bypass_mask: u8,
) -> Result<StalkerCheckpoint, TuliproxError> {
if let Some(state) = load_checkpoint(storage_path, identity_fingerprint).await? {
if let Some(mut state) = load_checkpoint(storage_path, identity_fingerprint).await? {
if state.selection_mask == selection.mask() {
let combined_bypass_mask = state.quality_bypass_mask | quality_bypass_mask;
if combined_bypass_mask != state.quality_bypass_mask {
state.quality_bypass_mask = combined_bypass_mask;
save_checkpoint(storage_path, &state).await?;
}
return Ok(state);
}
}
@@ -179,30 +229,70 @@ async fn load_or_start_checkpoint(
let mut state =
StalkerCheckpoint::new(identity_fingerprint, generation_id(), selection.mask(), chrono::Utc::now().timestamp());
state.phase = first_phase(selection);
state.quality_bypass_mask = quality_bypass_mask;
save_checkpoint(storage_path, &state).await?;
Ok(state)
}
async fn finish_completed_refresh_checked(
async fn evaluate_completed_refresh_publication(
app_config: &Arc<AppConfig>,
storage_path: &Path,
identity_fingerprint: u64,
checkpoint: &StalkerCheckpoint,
update_quality: ConfigInputUpdateQuality,
) -> Result<StalkerGenerationPublication, TuliproxError> {
let needs_quality_evaluation =
selection_needs_quality_evaluation(checkpoint.selection_mask, checkpoint.quality_bypass_mask, update_quality);
let has_quality_bypass = checkpoint.quality_bypass_mask & MEDIA_SELECTION != 0;
if needs_quality_evaluation || has_quality_bypass {
let active_manifest = if needs_quality_evaluation {
Some(load_active_manifest(storage_path, identity_fingerprint).await?)
} else {
None
};
evaluate_generation_publication(
app_config,
storage_path,
checkpoint.generation,
checkpoint.selection_mask,
checkpoint.quality_bypass_mask,
update_quality,
active_manifest.as_ref(),
)
.await
} else {
Ok(StalkerGenerationPublication {
accepted_selection_mask: checkpoint.selection_mask,
quality_acceptances: Vec::new(),
quality_rejections: Vec::new(),
force_updates: Vec::new(),
})
}
}
async fn ensure_completed_refresh_has_usable_playlist(
app_config: &Arc<AppConfig>,
storage_path: &Path,
identity_fingerprint: u64,
checkpoint: &StalkerCheckpoint,
accepted_selection_mask: u8,
) -> Result<(), TuliproxError> {
if checkpoint.processed == 0 {
let accepted_media_selection = accepted_selection_mask & MEDIA_SELECTION;
let accepted_enforced_selection = accepted_media_selection & !checkpoint.quality_bypass_mask;
if checkpoint.processed == 0 && accepted_enforced_selection != 0 {
let active = load_active_manifest(storage_path, identity_fingerprint).await?;
let mut retained_has_items = false;
if checkpoint.selection_mask & 0b0001 == 0 {
if accepted_selection_mask & LIVE_SELECTION == 0 {
if let Some(files) = active.live.as_ref() {
retained_has_items = !load_stalker_items_after(app_config, &files.data, None, 1).await?.is_empty();
}
}
if !retained_has_items && checkpoint.selection_mask & 0b0010 == 0 {
if !retained_has_items && accepted_selection_mask & VOD_SELECTION == 0 {
if let Some(files) = active.vod.as_ref() {
retained_has_items = !load_stalker_items_after(app_config, &files.data, None, 1).await?.is_empty();
}
}
if !retained_has_items && checkpoint.selection_mask & 0b0100 == 0 {
if !retained_has_items && accepted_selection_mask & SERIES_SELECTION == 0 {
if let Some(files) = active.series.as_ref() {
retained_has_items = !load_stalker_items_after(app_config, &files.roots, None, 1).await?.is_empty()
|| !load_stalker_items_after(app_config, &files.episodes, None, 1).await?.is_empty();
@@ -216,18 +306,242 @@ async fn finish_completed_refresh_checked(
));
}
}
finish_completed_refresh(storage_path, identity_fingerprint, checkpoint).await
Ok(())
}
async fn finish_completed_refresh(
async fn publish_completed_refresh(
storage_path: &Path,
identity_fingerprint: u64,
checkpoint: &StalkerCheckpoint,
accepted_selection_mask: u8,
) -> Result<(), TuliproxError> {
let manifest =
publish_selection(storage_path, identity_fingerprint, checkpoint.generation, checkpoint.selection_mask).await?;
publish_selection(storage_path, identity_fingerprint, checkpoint.generation, accepted_selection_mask).await?;
clear_checkpoint(storage_path).await?;
cleanup_obsolete_generations(storage_path, &manifest).await
cleanup_obsolete_generations(storage_path, &manifest).await?;
Ok(())
}
struct StalkerRefreshReports {
quality_acceptances: Vec<ClusterUpdateAcceptance>,
quality_rejections: Vec<ClusterUpdateRejection>,
force_updates: Vec<ClusterForceUpdate>,
error: Option<TuliproxError>,
}
impl StalkerRefreshReports {
fn into_outcome(self) -> StalkerRefreshOutcome {
let Self { quality_acceptances, quality_rejections, force_updates, error } = self;
if let Some(error) = error {
StalkerRefreshOutcome::Failed { quality_acceptances, quality_rejections, force_updates, error }
} else {
StalkerRefreshOutcome::Complete { quality_acceptances, quality_rejections, force_updates }
}
}
}
#[cfg(test)]
async fn finish_completed_refresh(
app_config: &Arc<AppConfig>,
storage_path: &Path,
identity_fingerprint: u64,
checkpoint: &StalkerCheckpoint,
update_quality: ConfigInputUpdateQuality,
) -> Result<Vec<ClusterUpdateRejection>, TuliproxError> {
let publication = evaluate_completed_refresh_publication(
app_config,
storage_path,
identity_fingerprint,
checkpoint,
update_quality,
)
.await?;
let StalkerGenerationPublication {
accepted_selection_mask,
quality_rejections,
quality_acceptances: _,
force_updates: _,
} = publication;
publish_completed_refresh(storage_path, identity_fingerprint, checkpoint, accepted_selection_mask).await?;
Ok(quality_rejections)
}
async fn finish_completed_refresh_checked(
app_config: &Arc<AppConfig>,
storage_path: &Path,
identity_fingerprint: u64,
checkpoint: &StalkerCheckpoint,
update_quality: ConfigInputUpdateQuality,
) -> Result<StalkerRefreshReports, TuliproxError> {
let publication = evaluate_completed_refresh_publication(
app_config,
storage_path,
identity_fingerprint,
checkpoint,
update_quality,
)
.await?;
let StalkerGenerationPublication {
accepted_selection_mask,
quality_acceptances,
quality_rejections,
force_updates,
} = publication;
let completion_result = async {
ensure_completed_refresh_has_usable_playlist(
app_config,
storage_path,
identity_fingerprint,
checkpoint,
accepted_selection_mask,
)
.await?;
publish_completed_refresh(storage_path, identity_fingerprint, checkpoint, accepted_selection_mask).await
}
.await;
Ok(StalkerRefreshReports { quality_acceptances, quality_rejections, force_updates, error: completion_result.err() })
}
struct StalkerGenerationPublication {
accepted_selection_mask: u8,
quality_acceptances: Vec<ClusterUpdateAcceptance>,
quality_rejections: Vec<ClusterUpdateRejection>,
force_updates: Vec<ClusterForceUpdate>,
}
fn selection_needs_quality_evaluation(
requested_selection_mask: u8,
quality_bypass_mask: u8,
update_quality: ConfigInputUpdateQuality,
) -> bool {
[StalkerCluster::Live, StalkerCluster::Vod, StalkerCluster::Series].into_iter().any(|cluster| {
let bit = selection_bit(cluster);
requested_selection_mask & bit != 0
&& quality_bypass_mask & bit == 0
&& update_quality.threshold(quality_cluster(cluster)) != 0
})
}
const fn selection_bit(cluster: StalkerCluster) -> u8 {
match cluster {
StalkerCluster::Live => LIVE_SELECTION,
StalkerCluster::Vod => VOD_SELECTION,
StalkerCluster::Series => SERIES_SELECTION,
}
}
const fn quality_cluster(cluster: StalkerCluster) -> XtreamCluster {
match cluster {
StalkerCluster::Live => XtreamCluster::Live,
StalkerCluster::Vod => XtreamCluster::Video,
StalkerCluster::Series => XtreamCluster::Series,
}
}
async fn count_stalker_cluster_files(
app_config: &Arc<AppConfig>,
primary: &Path,
secondary: Option<&Path>,
) -> Result<usize, TuliproxError> {
let primary_count = count_stalker_items_at(app_config, primary).await?.unwrap_or_default();
let secondary_count = match secondary {
Some(path) => count_stalker_items_at(app_config, path).await?.unwrap_or_default(),
None => 0,
};
primary_count.checked_add(secondary_count).ok_or_else(|| {
TuliproxError::RepositoryStalker("Stalker cluster item count exceeds platform capacity".to_string())
})
}
async fn count_candidate_cluster(
app_config: &Arc<AppConfig>,
storage_path: &Path,
generation: u64,
cluster: StalkerCluster,
) -> Result<usize, TuliproxError> {
let primary_data = match cluster {
StalkerCluster::Live => StalkerGenerationData::Live,
StalkerCluster::Vod => StalkerGenerationData::Vod,
StalkerCluster::Series => StalkerGenerationData::SeriesRoots,
};
let primary = generation_data_path(storage_path, generation, primary_data);
let secondary = (cluster == StalkerCluster::Series)
.then(|| generation_data_path(storage_path, generation, StalkerGenerationData::SeriesEpisodes));
count_stalker_cluster_files(app_config, &primary, secondary.as_deref()).await
}
async fn count_active_cluster(
app_config: &Arc<AppConfig>,
manifest: &tuliprox_repository::stalker_generation_repository::StalkerActiveManifest,
cluster: StalkerCluster,
) -> Result<Option<usize>, TuliproxError> {
match cluster {
StalkerCluster::Live => match manifest.live.as_ref() {
Some(files) => count_stalker_cluster_files(app_config, &files.data, None).await.map(Some),
None => Ok(None),
},
StalkerCluster::Vod => match manifest.vod.as_ref() {
Some(files) => count_stalker_cluster_files(app_config, &files.data, None).await.map(Some),
None => Ok(None),
},
StalkerCluster::Series => match manifest.series.as_ref() {
Some(files) => count_stalker_cluster_files(app_config, &files.roots, Some(&files.episodes)).await.map(Some),
None => Ok(None),
},
}
}
async fn evaluate_generation_publication(
app_config: &Arc<AppConfig>,
storage_path: &Path,
generation: u64,
requested_selection_mask: u8,
quality_bypass_mask: u8,
update_quality: ConfigInputUpdateQuality,
active_manifest: Option<&tuliprox_repository::stalker_generation_repository::StalkerActiveManifest>,
) -> Result<StalkerGenerationPublication, TuliproxError> {
let mut accepted_selection_mask = requested_selection_mask;
let mut quality_acceptances = Vec::new();
let mut quality_rejections = Vec::new();
let mut force_updates = Vec::new();
for cluster in [StalkerCluster::Live, StalkerCluster::Vod, StalkerCluster::Series] {
let selection_bit = selection_bit(cluster);
if requested_selection_mask & selection_bit == 0 {
continue;
}
let report_cluster = quality_cluster(cluster);
let threshold = update_quality.threshold(report_cluster);
if quality_bypass_mask & selection_bit != 0 {
let candidate_count = count_candidate_cluster(app_config, storage_path, generation, cluster).await?;
force_updates.push(ClusterForceUpdate {
cluster: report_cluster,
candidate_count,
configured_threshold: threshold,
});
continue;
}
if threshold == 0 {
continue;
}
let candidate_count = count_candidate_cluster(app_config, storage_path, generation, cluster).await?;
let Some(active_manifest) = active_manifest else {
return Err(TuliproxError::RepositoryStalker(format!(
"Missing active manifest for enforced {report_cluster} quality evaluation"
)));
};
let current_count = count_active_cluster(app_config, active_manifest, cluster).await?;
let decision = evaluate_update_quality(current_count, candidate_count, threshold);
if let Some(acceptance) = decision.acceptance(report_cluster) {
quality_acceptances.push(acceptance);
} else if let Some(rejection) = decision.rejection(report_cluster) {
accepted_selection_mask &= !selection_bit;
if cluster == StalkerCluster::Live {
accepted_selection_mask &= !EPG_SELECTION;
}
quality_rejections.push(rejection);
}
}
Ok(StalkerGenerationPublication { accepted_selection_mask, quality_acceptances, quality_rejections, force_updates })
}
fn category_map(categories: Vec<StalkerCategory>) -> HashMap<u32, StalkerCategory> {
@@ -317,12 +631,14 @@ pub async fn advance_stalker_refresh(
app_config: &Arc<AppConfig>,
api_client: &StalkerApiClient,
handshake: &StalkerHandshake,
selection: StalkerClusterSelection,
refresh_plan: StalkerRefreshPlan,
storage_path: &Path,
identity_fingerprint: u64,
mut budget: StalkerRefreshBudget,
) -> Result<StalkerRefreshOutcome, TuliproxError> {
let mut checkpoint = load_or_start_checkpoint(storage_path, identity_fingerprint, selection).await?;
let StalkerRefreshPlan { selection, update_quality, quality_bypass_mask } = refresh_plan;
let mut checkpoint =
load_or_start_checkpoint(storage_path, identity_fingerprint, selection, quality_bypass_mask).await?;
if checkpoint.phase == StalkerRefreshPhase::Terminal {
return finish_terminal_refresh(storage_path).await;
}
@@ -568,8 +884,15 @@ pub async fn advance_stalker_refresh(
checkpoint.retry_count = 0;
}
StalkerRefreshPhase::Complete => {
finish_completed_refresh_checked(app_config, storage_path, identity_fingerprint, &checkpoint).await?;
return Ok(StalkerRefreshOutcome::Complete);
let reports = finish_completed_refresh_checked(
app_config,
storage_path,
identity_fingerprint,
&checkpoint,
update_quality,
)
.await?;
return Ok(reports.into_outcome());
}
StalkerRefreshPhase::Terminal => {
return finish_terminal_refresh(storage_path).await;
@@ -584,11 +907,17 @@ pub async fn advance_stalker_refresh(
mod tests {
use super::*;
use arc_swap::{ArcSwap, ArcSwapOption};
use shared::model::ConfigPaths;
use shared::{
model::{ConfigInputUpdateQualityDto, ConfigPaths},
utils::Internable,
};
use tuliprox_core::{
model::{ApiProxyConfig, Config, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig},
utils::FileLockManager,
};
use tuliprox_repository::stalker_generation_repository::{
load_active_manifest, save_active_manifest, ClusterFiles, SeriesFiles, StalkerActiveManifest,
};
fn test_app_config(storage_dir: &Path) -> Arc<AppConfig> {
Arc::new(AppConfig {
@@ -620,6 +949,78 @@ mod tests {
})
}
fn update_quality(live: u8, vod: u8, series: u8) -> ConfigInputUpdateQuality {
ConfigInputUpdateQuality::from(&ConfigInputUpdateQualityDto { live, vod, series })
}
fn stalker_items(count: usize, kind: StalkerStreamKind, series_roots: bool) -> Vec<StalkerPlaylistItem> {
(0..count)
.map(|index| StalkerPlaylistItem {
stream_id: u32::try_from(index + 1).unwrap_or(u32::MAX),
name: format!("item-{index}").intern(),
stream_kind: kind,
is_series: series_roots,
..StalkerPlaylistItem::default()
})
.collect()
}
async fn write_generation_cluster(
app_config: &Arc<AppConfig>,
storage_path: &Path,
generation: u64,
cluster: StalkerCluster,
primary_count: usize,
episode_count: usize,
) -> Result<(), TuliproxError> {
let (primary_data, kind, series_roots) = match cluster {
StalkerCluster::Live => (StalkerGenerationData::Live, StalkerStreamKind::Live, false),
StalkerCluster::Vod => (StalkerGenerationData::Vod, StalkerStreamKind::Movie, false),
StalkerCluster::Series => (StalkerGenerationData::SeriesRoots, StalkerStreamKind::Episode, true),
};
let primary_path = generation_data_path(storage_path, generation, primary_data);
let primary_items = stalker_items(primary_count, kind, series_roots);
if primary_items.is_empty() {
upsert_stalker_items_at(app_config, &primary_path, &primary_items).await?;
} else {
snapshot_stalker_items_at(app_config, primary_path, &primary_items).await?;
}
if cluster == StalkerCluster::Series {
let episode_path = generation_data_path(storage_path, generation, StalkerGenerationData::SeriesEpisodes);
snapshot_stalker_items_at(
app_config,
episode_path,
&stalker_items(episode_count, StalkerStreamKind::Episode, false),
)
.await?;
}
Ok(())
}
fn complete_manifest(storage_path: &Path, identity_fingerprint: u64, generation: u64) -> StalkerActiveManifest {
StalkerActiveManifest {
schema: StalkerActiveManifest::empty(identity_fingerprint).schema,
identity_fingerprint,
live: Some(ClusterFiles {
generation,
data: generation_data_path(storage_path, generation, StalkerGenerationData::Live),
}),
vod: Some(ClusterFiles {
generation,
data: generation_data_path(storage_path, generation, StalkerGenerationData::Vod),
}),
series: Some(SeriesFiles {
generation,
roots: generation_data_path(storage_path, generation, StalkerGenerationData::SeriesRoots),
episodes: generation_data_path(storage_path, generation, StalkerGenerationData::SeriesEpisodes),
}),
epg: Some(ClusterFiles {
generation,
data: generation_data_path(storage_path, generation, StalkerGenerationData::Epg),
}),
}
}
#[tokio::test]
async fn terminal_checkpoint_is_cleared_after_restart() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
@@ -643,13 +1044,33 @@ mod tests {
checkpoint.phase = StalkerRefreshPhase::Complete;
save_checkpoint(temp.path(), &checkpoint).await?;
let resumed = load_or_start_checkpoint(temp.path(), 17, selection).await?;
let resumed = load_or_start_checkpoint(temp.path(), 17, selection, 0).await?;
assert_eq!(resumed.generation, 23);
assert_eq!(resumed.phase, StalkerRefreshPhase::Complete);
Ok(())
}
#[tokio::test]
async fn resumed_checkpoint_preserves_and_extends_the_quality_bypass_mask() -> Result<(), Box<dyn std::error::Error>>
{
let temp = tempfile::tempdir()?;
let selection = StalkerClusterSelection { live: true, vod: true, series: true, epg: false };
let mut checkpoint = StalkerCheckpoint::new(17, 23, selection.mask(), 123);
checkpoint.phase = StalkerRefreshPhase::Vod { page: 4 };
checkpoint.quality_bypass_mask = LIVE_SELECTION;
save_checkpoint(temp.path(), &checkpoint).await?;
let resumed = load_or_start_checkpoint(temp.path(), 17, selection, SERIES_SELECTION).await?;
assert_eq!(resumed.generation, 23);
assert_eq!(resumed.phase, StalkerRefreshPhase::Vod { page: 4 });
assert_eq!(resumed.quality_bypass_mask, LIVE_SELECTION | SERIES_SELECTION);
let resumed_without_new_force = load_or_start_checkpoint(temp.path(), 17, selection, 0).await?;
assert_eq!(resumed_without_new_force.quality_bypass_mask, LIVE_SELECTION | SERIES_SELECTION);
Ok(())
}
#[tokio::test]
async fn empty_complete_refresh_keeps_the_active_manifest() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
@@ -666,9 +1087,16 @@ mod tests {
checkpoint.phase = StalkerRefreshPhase::Complete;
save_checkpoint(temp.path(), &checkpoint).await?;
let result = finish_completed_refresh_checked(&app_config, temp.path(), 17, &checkpoint).await;
let reports = finish_completed_refresh_checked(
&app_config,
temp.path(),
17,
&checkpoint,
ConfigInputUpdateQuality::default(),
)
.await?;
assert!(result.is_err());
assert!(reports.error.is_some());
assert_eq!(load_active_manifest(temp.path(), 17).await?, active);
assert!(load_checkpoint(temp.path(), 17).await?.is_none());
Ok(())
@@ -677,22 +1105,368 @@ mod tests {
#[tokio::test]
async fn complete_publication_is_idempotent_after_restart() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let mut checkpoint = StalkerCheckpoint::new(17, 23, 0b0011, 123);
checkpoint.phase = StalkerRefreshPhase::Complete;
save_checkpoint(temp.path(), &checkpoint).await?;
finish_completed_refresh(temp.path(), 17, &checkpoint).await?;
finish_completed_refresh(&app_config, temp.path(), 17, &checkpoint, ConfigInputUpdateQuality::default())
.await?;
save_checkpoint(temp.path(), &checkpoint).await?;
finish_completed_refresh(temp.path(), 17, &checkpoint).await?;
finish_completed_refresh(&app_config, temp.path(), 17, &checkpoint, ConfigInputUpdateQuality::default())
.await?;
let manifest =
tuliprox_repository::stalker_generation_repository::load_active_manifest(temp.path(), 17).await?;
let manifest = load_active_manifest(temp.path(), 17).await?;
assert_eq!(manifest.live.as_ref().map(|files| files.generation), Some(23));
assert_eq!(manifest.vod.as_ref().map(|files| files.generation), Some(23));
assert!(load_checkpoint(temp.path(), 17).await?.is_none());
Ok(())
}
#[tokio::test]
async fn pipeline_transparency_stalker_preserves_accepted_quality_facts_after_publication(
) -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 17;
let active_generation = 41;
let candidate_generation = 42;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Vod, 100, 0).await?;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Series, 40, 60).await?;
let active = complete_manifest(temp.path(), identity, active_generation);
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 90, 0).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Vod, 89, 0).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Series, 45, 65)
.await?;
let checkpoint = StalkerCheckpoint::new(identity, candidate_generation, 0b1111, 123);
let reports = finish_completed_refresh_checked(
&app_config,
temp.path(),
identity,
&checkpoint,
update_quality(90, 90, 90),
)
.await?;
assert!(reports.error.is_none());
assert_eq!(
reports.quality_acceptances,
vec![
ClusterUpdateAcceptance {
cluster: XtreamCluster::Live,
current_count: Some(100),
candidate_count: 90,
threshold: 90,
quality: Some(90),
},
ClusterUpdateAcceptance {
cluster: XtreamCluster::Series,
current_count: Some(100),
candidate_count: 110,
threshold: 90,
quality: Some(90),
},
]
);
assert_eq!(
reports.quality_rejections,
vec![ClusterUpdateRejection {
cluster: XtreamCluster::Video,
current_count: 100,
candidate_count: 89,
threshold: 90,
quality: 89,
}]
);
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
assert_eq!(published.vod, active.vod);
assert_eq!(published.series.as_ref().map(|files| files.generation), Some(candidate_generation));
assert_eq!(published.epg.as_ref().map(|files| files.generation), Some(candidate_generation));
Ok(())
}
#[tokio::test]
async fn pipeline_transparency_stalker_keeps_acceptance_when_pre_publish_validation_fails(
) -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 18;
let active_generation = 51;
let candidate_generation = 52;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
let mut active = StalkerActiveManifest::empty(identity);
active.live = Some(ClusterFiles {
generation: active_generation,
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
});
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 90, 0).await?;
let mut checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION, 123);
checkpoint.phase = StalkerRefreshPhase::Complete;
save_checkpoint(temp.path(), &checkpoint).await?;
let reports =
finish_completed_refresh_checked(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0))
.await?;
let StalkerRefreshOutcome::Failed { quality_acceptances, error: _, .. } = reports.into_outcome() else {
return Err("expected failed Stalker completion with retained reports".into());
};
assert_eq!(
quality_acceptances,
vec![ClusterUpdateAcceptance {
cluster: XtreamCluster::Live,
current_count: Some(100),
candidate_count: 90,
threshold: 90,
quality: Some(90),
}]
);
assert_eq!(load_active_manifest(temp.path(), identity).await?, active);
assert!(load_checkpoint(temp.path(), identity).await?.is_none());
Ok(())
}
#[tokio::test]
async fn pipeline_transparency_stalker_keeps_acceptance_when_cleanup_fails_after_publish(
) -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 19;
let active_generation = 61;
let candidate_generation = 62;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
let mut active = StalkerActiveManifest::empty(identity);
active.live = Some(ClusterFiles {
generation: active_generation,
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
});
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 90, 0).await?;
tokio::fs::create_dir(generation_data_path(temp.path(), 1, StalkerGenerationData::Live)).await?;
tokio::fs::write(generation_data_path(temp.path(), 2, StalkerGenerationData::Live), b"obsolete").await?;
let mut checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION, 123);
checkpoint.phase = StalkerRefreshPhase::Complete;
checkpoint.processed = 90;
save_checkpoint(temp.path(), &checkpoint).await?;
let reports =
finish_completed_refresh_checked(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0))
.await?;
let StalkerRefreshOutcome::Failed { quality_acceptances, error: _, .. } = reports.into_outcome() else {
return Err("expected failed Stalker completion with retained reports".into());
};
assert_eq!(quality_acceptances.len(), 1);
assert_eq!(quality_acceptances[0].cluster, XtreamCluster::Live);
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
assert!(load_checkpoint(temp.path(), identity).await?.is_none());
Ok(())
}
#[tokio::test]
async fn rejected_live_generation_keeps_active_live_and_epg() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 23;
let active_generation = 7;
let candidate_generation = 8;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
let mut active = StalkerActiveManifest::empty(identity);
active.live = Some(ClusterFiles {
generation: active_generation,
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
});
active.epg = Some(ClusterFiles {
generation: active_generation,
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Epg),
});
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 89, 0).await?;
let checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION | EPG_SELECTION, 123);
let rejections =
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0)).await?;
assert_eq!(rejections.len(), 1);
assert_eq!(rejections[0].cluster, XtreamCluster::Live);
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.live, active.live);
assert_eq!(published.epg, active.epg);
Ok(())
}
#[tokio::test]
async fn nonempty_bootstrap_generation_is_published() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 29;
let generation = 3;
write_generation_cluster(&app_config, temp.path(), generation, StalkerCluster::Series, 2, 3).await?;
let checkpoint = StalkerCheckpoint::new(identity, generation, SERIES_SELECTION, 123);
let rejections =
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(0, 0, 100))
.await?;
assert!(rejections.is_empty());
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.series.as_ref().map(|files| files.generation), Some(generation));
Ok(())
}
#[tokio::test]
async fn empty_bootstrap_generation_is_rejected() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 30;
let generation = 4;
write_generation_cluster(&app_config, temp.path(), generation, StalkerCluster::Live, 0, 0).await?;
let checkpoint = StalkerCheckpoint::new(identity, generation, LIVE_SELECTION | EPG_SELECTION, 123);
let rejections =
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0)).await?;
assert_eq!(
rejections,
vec![ClusterUpdateRejection {
cluster: XtreamCluster::Live,
current_count: 0,
candidate_count: 0,
threshold: 90,
quality: 0,
}]
);
let published = load_active_manifest(temp.path(), identity).await?;
assert!(published.live.is_none());
assert!(published.epg.is_none());
Ok(())
}
#[tokio::test]
async fn forced_empty_stalker_generation_is_published_without_a_quality_rejection(
) -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 40;
let active_generation = 5;
let candidate_generation = 6;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Vod, 10, 0).await?;
let mut active = complete_manifest(temp.path(), identity, active_generation);
active.live = None;
active.series = None;
active.epg = None;
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Vod, 0, 0).await?;
let mut checkpoint = StalkerCheckpoint::new(identity, candidate_generation, VOD_SELECTION, 123);
checkpoint.phase = StalkerRefreshPhase::Complete;
checkpoint.quality_bypass_mask = VOD_SELECTION;
save_checkpoint(temp.path(), &checkpoint).await?;
let reports = finish_completed_refresh_checked(
&app_config,
temp.path(),
identity,
&checkpoint,
update_quality(0, 100, 0),
)
.await?;
assert!(reports.quality_rejections.is_empty());
assert_eq!(
reports.force_updates,
vec![ClusterForceUpdate { cluster: XtreamCluster::Video, candidate_count: 0, configured_threshold: 100 }]
);
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.vod.as_ref().map(|files| files.generation), Some(candidate_generation));
assert!(load_checkpoint(temp.path(), identity).await?.is_none());
Ok(())
}
#[tokio::test]
async fn forced_live_generation_publishes_its_epg_with_the_same_generation(
) -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 41;
let active_generation = 7;
let candidate_generation = 8;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 10, 0).await?;
let mut active = StalkerActiveManifest::empty(identity);
active.live = Some(ClusterFiles {
generation: active_generation,
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
});
active.epg = Some(ClusterFiles {
generation: active_generation,
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Epg),
});
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 1, 0).await?;
let mut checkpoint =
StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION | EPG_SELECTION, 123);
checkpoint.quality_bypass_mask = LIVE_SELECTION;
let reports = finish_completed_refresh_checked(
&app_config,
temp.path(),
identity,
&checkpoint,
update_quality(100, 0, 0),
)
.await?;
assert!(reports.quality_rejections.is_empty());
assert_eq!(reports.force_updates.len(), 1);
assert_eq!(reports.force_updates[0].cluster, XtreamCluster::Live);
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
assert_eq!(published.epg.as_ref().map(|files| files.generation), Some(candidate_generation));
Ok(())
}
#[tokio::test]
async fn rejected_series_counts_roots_and_episodes_without_blocking_accepted_live_generation(
) -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let app_config = test_app_config(temp.path());
let identity = 31;
let active_generation = 11;
let candidate_generation = 12;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 10, 0).await?;
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Series, 2, 8).await?;
let mut active = complete_manifest(temp.path(), identity, active_generation);
active.vod = None;
save_active_manifest(temp.path(), &active).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 10, 0).await?;
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Series, 2, 7).await?;
let checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION | SERIES_SELECTION, 123);
assert_eq!(count_active_cluster(&app_config, &active, StalkerCluster::Series).await?, Some(10));
assert_eq!(
count_candidate_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Series).await?,
9
);
let rejections =
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(100, 0, 100))
.await?;
assert_eq!(rejections.len(), 1);
assert_eq!(rejections[0].cluster, XtreamCluster::Series);
assert_eq!(rejections[0].current_count, 10);
assert_eq!(rejections[0].candidate_count, 9);
let published = load_active_manifest(temp.path(), identity).await?;
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
assert_eq!(published.series, active.series);
Ok(())
}
#[test]
fn unit_budget_yields_after_completed_unit() {
let mut budget = StalkerRefreshBudget::units(1);