mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-19 22:32:26 +02:00
feat(update): add per-cluster quality safeguards (#855)
* feat(update): add per-input actions and pipeline transparency Squash feature/cluster-update-quality onto upstream/develop. Include request-local update policies, persisted input and cluster status, library rescan and target rebuild integration, and the update-card view. Align the M3U request regression with the existing Force capability. * del(build): temporary dev changes * fix(update): address playlist review findings * fix(update): rename Shows to Series * Playlist update refactor * Playlist update refactor --------- Co-authored-by: euzu <euzu@proton.me>
This commit is contained in:
@@ -66,16 +66,19 @@ pub async fn download_library_playlist(
|
||||
input: &ConfigInput,
|
||||
) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
|
||||
let config = &*app_config.config.load();
|
||||
let Some(library_config) = config.library.as_ref() else { return (vec![], vec![]) };
|
||||
if !library_config.enabled {
|
||||
return (vec![], vec![]);
|
||||
}
|
||||
let failure = |error: String| (vec![], vec![TuliproxError::RepositoryLibrary(error)]);
|
||||
let Some(library_config) = config.library.as_ref().filter(|library| library.enabled) else {
|
||||
return failure("Library is disabled or unavailable".into());
|
||||
};
|
||||
let api_base_path =
|
||||
concat_path_leading_slash(config.web_ui.as_ref().and_then(|w| w.path.as_deref()).unwrap_or(""), "api/v1");
|
||||
|
||||
let storage_path =
|
||||
resolve_metadata_storage_path(config.metadata_update.as_ref(), &config.storage_dir).join("library");
|
||||
let mut metadata_iter = MetadataAsyncIter::new(&storage_path).await;
|
||||
let mut metadata_iter = match MetadataAsyncIter::try_new(&storage_path).await {
|
||||
Ok(iter) => iter,
|
||||
Err(err) => return failure(format!("Failed to open Library catalog: {err}")),
|
||||
};
|
||||
let mut group_movies = PlaylistGroup {
|
||||
id: 0,
|
||||
title: library_config.playlist.movie_category.clone(),
|
||||
@@ -88,7 +91,12 @@ pub async fn download_library_playlist(
|
||||
channels: vec![],
|
||||
xtream_cluster: XtreamCluster::Series,
|
||||
};
|
||||
while let Some(entry) = metadata_iter.next().await {
|
||||
loop {
|
||||
let entry = match metadata_iter.try_next().await {
|
||||
Ok(Some(entry)) => entry,
|
||||
Ok(None) => break,
|
||||
Err(err) => return failure(format!("Failed to read Library catalog: {err}")),
|
||||
};
|
||||
match entry.metadata {
|
||||
MediaMetadata::Movie(_) => {
|
||||
to_playlist_item(
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
use crate::input_cache::{self, ClusterState, InputStatus};
|
||||
use log::warn;
|
||||
use shared::model::{
|
||||
EventMessage, EventSink, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, XtreamCluster,
|
||||
};
|
||||
use tuliprox_core::model::{ClusterForceUpdate, ClusterUpdateRejection, ConfigInput};
|
||||
use tuliprox_iptv::provider::PlaylistFetch;
|
||||
|
||||
/// Cache entries whose state is determined by one completed provider fetch.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum CacheStatusScope<'a> {
|
||||
Default,
|
||||
RequestedClusters(&'a [XtreamCluster]),
|
||||
}
|
||||
|
||||
/// Applies the observable and cache-state effects of a completed provider fetch.
|
||||
///
|
||||
/// Quality rejections remain separate from technical errors. For per-cluster
|
||||
/// fetches, only requested clusters are updated, so valid cached clusters keep
|
||||
/// both their state and timestamp.
|
||||
pub(super) fn apply_playlist_fetch_outcome<E: EventSink>(
|
||||
events: &E,
|
||||
run_id: &PlaylistUpdateRunId,
|
||||
execution_order: PlaylistUpdateRunOrder,
|
||||
input: &ConfigInput,
|
||||
status: &mut InputStatus,
|
||||
cache_scope: CacheStatusScope<'_>,
|
||||
fetch: &PlaylistFetch,
|
||||
) -> bool {
|
||||
report_quality_rejections(events, run_id, execution_order, input, &fetch.quality_rejections);
|
||||
if fetch.persisted {
|
||||
report_force_updates(events, run_id, execution_order, input, &fetch.force_updates);
|
||||
}
|
||||
|
||||
let technical_failure = fetch.partial || !fetch.errors.is_empty();
|
||||
match cache_scope {
|
||||
CacheStatusScope::RequestedClusters(requested_clusters) => {
|
||||
for cluster in requested_clusters {
|
||||
let rejected = fetch.quality_rejections.iter().any(|rejection| rejection.cluster == *cluster);
|
||||
let state = if technical_failure || rejected { ClusterState::Failed } else { ClusterState::Ok };
|
||||
input_cache::update_cluster_status(status, cluster.as_ref(), state);
|
||||
}
|
||||
!requested_clusters.is_empty()
|
||||
}
|
||||
CacheStatusScope::Default => {
|
||||
let default_state = if technical_failure || !fetch.quality_rejections.is_empty() {
|
||||
ClusterState::Failed
|
||||
} else {
|
||||
ClusterState::Ok
|
||||
};
|
||||
input_cache::update_cluster_status(status, "default", default_state);
|
||||
|
||||
// M3U still acquires/caches one document, but its completed decisions
|
||||
// and reload snapshots are per cluster. A rejected cluster invalidates
|
||||
// the document cache without labelling accepted siblings Failed.
|
||||
if input.get_download_input_type() == shared::model::InputType::M3u {
|
||||
for cluster in super::ingest::PIPELINE_TRANSPARENCY_CLUSTERS {
|
||||
if super::ingest::cluster_is_configured(input, cluster) {
|
||||
let rejected = fetch.quality_rejections.iter().any(|item| item.cluster == cluster);
|
||||
let state = if technical_failure || rejected { ClusterState::Failed } else { ClusterState::Ok };
|
||||
input_cache::update_cluster_status(status, cluster.as_ref(), state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for rejection in &fetch.quality_rejections {
|
||||
input_cache::update_cluster_status(status, rejection.cluster.as_ref(), ClusterState::Failed);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn report_force_updates<E: EventSink>(
|
||||
events: &E,
|
||||
run_id: &PlaylistUpdateRunId,
|
||||
execution_order: PlaylistUpdateRunOrder,
|
||||
input: &ConfigInput,
|
||||
force_updates: &[ClusterForceUpdate],
|
||||
) {
|
||||
for force_update in force_updates {
|
||||
let message = format!(
|
||||
"Input '{}' cluster '{}' force-published: candidate={} configured_threshold={}",
|
||||
input.name,
|
||||
quality_cluster_name(force_update.cluster),
|
||||
force_update.candidate_count,
|
||||
force_update.configured_threshold
|
||||
);
|
||||
log::info!("{message}");
|
||||
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input(
|
||||
run_id.clone(),
|
||||
execution_order,
|
||||
input.id,
|
||||
input.name.to_string(),
|
||||
message,
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
fn report_quality_rejections<E: EventSink>(
|
||||
events: &E,
|
||||
run_id: &PlaylistUpdateRunId,
|
||||
execution_order: PlaylistUpdateRunOrder,
|
||||
input: &ConfigInput,
|
||||
rejections: &[ClusterUpdateRejection],
|
||||
) {
|
||||
for rejection in rejections {
|
||||
let message = format!(
|
||||
"Input '{}' cluster '{}' rejected: current={} candidate={} threshold={} quality={}; retaining previous cluster",
|
||||
input.name,
|
||||
quality_cluster_name(rejection.cluster),
|
||||
rejection.current_count,
|
||||
rejection.candidate_count,
|
||||
rejection.threshold,
|
||||
rejection.quality
|
||||
);
|
||||
warn!("{message}");
|
||||
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input(
|
||||
run_id.clone(),
|
||||
execution_order,
|
||||
input.id,
|
||||
input.name.to_string(),
|
||||
message,
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
const fn quality_cluster_name(cluster: XtreamCluster) -> &'static str {
|
||||
match cluster {
|
||||
XtreamCluster::Live => "live",
|
||||
XtreamCluster::Video => "vod",
|
||||
XtreamCluster::Series => "series",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{apply_playlist_fetch_outcome, CacheStatusScope};
|
||||
use crate::input_cache::{is_cache_valid, ClusterState, ClusterStatus, InputStatus};
|
||||
use shared::model::{
|
||||
EventMessage, EventSink, InputType, PlaylistUpdateRunId, PlaylistUpdateRunOrder, XtreamCluster,
|
||||
};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tuliprox_core::model::{ClusterForceUpdate, ClusterUpdateRejection, ConfigInput};
|
||||
use tuliprox_iptv::provider::PlaylistFetch;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CollectSink(Arc<Mutex<Vec<EventMessage>>>);
|
||||
|
||||
impl EventSink for CollectSink {
|
||||
fn emit(&self, event: EventMessage) { self.0.lock().expect("event sink lock").push(event); }
|
||||
}
|
||||
|
||||
fn rejection(cluster: XtreamCluster) -> ClusterUpdateRejection {
|
||||
ClusterUpdateRejection { cluster, current_count: 12_543, candidate_count: 217, threshold: 90, quality: 1 }
|
||||
}
|
||||
|
||||
fn run_id() -> PlaylistUpdateRunId { "fetch-outcome-run".into() }
|
||||
|
||||
fn execution_order() -> PlaylistUpdateRunOrder { PlaylistUpdateRunOrder::from(41) }
|
||||
|
||||
#[test]
|
||||
fn playlist_update_run_mixed_cluster_fetch_reports_correlated_rejection() {
|
||||
let input = ConfigInput { name: Arc::from("provider-a"), input_type: InputType::Xtream, ..Default::default() };
|
||||
let events = CollectSink::default();
|
||||
let mut status = InputStatus::default();
|
||||
status
|
||||
.clusters
|
||||
.insert("series".to_string(), ClusterStatus { status: ClusterState::Ok, timestamp: 17, last_update: None });
|
||||
let fetch = PlaylistFetch::groups(Vec::new()).with_quality_rejections(vec![rejection(XtreamCluster::Video)]);
|
||||
|
||||
let changed = apply_playlist_fetch_outcome(
|
||||
&events,
|
||||
&run_id(),
|
||||
execution_order(),
|
||||
&input,
|
||||
&mut status,
|
||||
CacheStatusScope::RequestedClusters(&[XtreamCluster::Live, XtreamCluster::Video]),
|
||||
&fetch,
|
||||
);
|
||||
|
||||
assert!(changed);
|
||||
assert_eq!(
|
||||
status.clusters.get(XtreamCluster::Live.as_ref()).map(|entry| &entry.status),
|
||||
Some(&ClusterState::Ok)
|
||||
);
|
||||
assert_eq!(
|
||||
status.clusters.get(XtreamCluster::Video.as_ref()).map(|entry| &entry.status),
|
||||
Some(&ClusterState::Failed)
|
||||
);
|
||||
let series = status.clusters.get("series").expect("cached series status");
|
||||
assert_eq!(series.status, ClusterState::Ok);
|
||||
assert_eq!(series.timestamp, 17);
|
||||
assert!(!status.clusters.contains_key("default"));
|
||||
|
||||
let emitted = events.0.lock().expect("event sink lock");
|
||||
assert_eq!(emitted.len(), 1);
|
||||
let EventMessage::PlaylistUpdateProgress(progress) = &emitted[0] else {
|
||||
panic!("expected playlist update progress event");
|
||||
};
|
||||
assert_eq!(progress.run_id.as_ref().map(AsRef::as_ref), Some("fetch-outcome-run"));
|
||||
assert_eq!(progress.execution_order, Some(execution_order()));
|
||||
assert_eq!(progress.target, "provider-a");
|
||||
assert_eq!(
|
||||
progress.message,
|
||||
"Input 'provider-a' cluster 'vod' rejected: current=12543 candidate=217 threshold=90 quality=1; retaining previous cluster"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_default_fetch_invalidates_default_and_cluster_cache_after_rejection() {
|
||||
let input = ConfigInput { name: Arc::from("provider-a"), input_type: InputType::Stalker, ..Default::default() };
|
||||
let events = CollectSink::default();
|
||||
let mut status = InputStatus::default();
|
||||
status.clusters.insert(
|
||||
"default".to_string(),
|
||||
ClusterStatus { status: ClusterState::Ok, timestamp: 23, last_update: None },
|
||||
);
|
||||
let fetch = PlaylistFetch::groups(Vec::new()).with_quality_rejections(vec![rejection(XtreamCluster::Series)]);
|
||||
|
||||
let changed = apply_playlist_fetch_outcome(
|
||||
&events,
|
||||
&run_id(),
|
||||
execution_order(),
|
||||
&input,
|
||||
&mut status,
|
||||
CacheStatusScope::Default,
|
||||
&fetch,
|
||||
);
|
||||
|
||||
assert!(changed);
|
||||
let default = status.clusters.get("default").expect("default cache status");
|
||||
assert_eq!(default.status, ClusterState::Failed);
|
||||
assert!(!is_cache_valid(&status, "default", u64::MAX));
|
||||
assert_eq!(
|
||||
status.clusters.get(XtreamCluster::Series.as_ref()).map(|entry| &entry.status),
|
||||
Some(&ClusterState::Failed)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn playlist_update_run_force_publication_emits_correlated_audit_progress() {
|
||||
let input = ConfigInput { name: Arc::from("provider-a"), input_type: InputType::Xtream, ..Default::default() };
|
||||
let events = CollectSink::default();
|
||||
let mut status = InputStatus::default();
|
||||
status.clusters.insert(
|
||||
XtreamCluster::Video.as_ref().to_string(),
|
||||
ClusterStatus { status: ClusterState::Failed, timestamp: 17, last_update: None },
|
||||
);
|
||||
let fetch = PlaylistFetch::default()
|
||||
.with_force_updates(vec![ClusterForceUpdate {
|
||||
cluster: XtreamCluster::Video,
|
||||
candidate_count: 12_450,
|
||||
configured_threshold: 95,
|
||||
}])
|
||||
.persisted(true);
|
||||
|
||||
let changed = apply_playlist_fetch_outcome(
|
||||
&events,
|
||||
&run_id(),
|
||||
execution_order(),
|
||||
&input,
|
||||
&mut status,
|
||||
CacheStatusScope::RequestedClusters(&[XtreamCluster::Video]),
|
||||
&fetch,
|
||||
);
|
||||
|
||||
assert!(changed);
|
||||
assert_eq!(
|
||||
status.clusters.get(XtreamCluster::Video.as_ref()).map(|entry| &entry.status),
|
||||
Some(&ClusterState::Ok)
|
||||
);
|
||||
let emitted = events.0.lock().expect("event sink lock");
|
||||
assert_eq!(emitted.len(), 1);
|
||||
let EventMessage::PlaylistUpdateProgress(progress) = &emitted[0] else {
|
||||
panic!("expected playlist update progress event");
|
||||
};
|
||||
assert_eq!(progress.run_id.as_ref().map(AsRef::as_ref), Some("fetch-outcome-run"));
|
||||
assert_eq!(progress.execution_order, Some(execution_order()));
|
||||
assert_eq!(progress.target, "provider-a");
|
||||
assert_eq!(
|
||||
progress.message,
|
||||
"Input 'provider-a' cluster 'vod' force-published: candidate=12450 configured_threshold=95"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,75 @@
|
||||
use super::{InputDownloadResult, InputJobResult, MetadataUpdateSink, PlaylistProcessingContext};
|
||||
use crate::input_cache::{load_input_status, resolve_input_storage_path, save_input_status};
|
||||
use shared::model::{EventSink, PersistedPlaylistUpdateInputResult, PlaylistUpdateState};
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
pub(super) async fn persist_input_job_result<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
|
||||
ctx: &PlaylistProcessingContext<E, M>,
|
||||
result: &InputJobResult,
|
||||
) {
|
||||
// Successful in-run reuse has no new acquisition facts and must not replace the
|
||||
// original result (in particular Partial) or advance its completion timestamp.
|
||||
let state = result.update_state();
|
||||
if result.input_telemetry.is_some() || state == PlaylistUpdateState::Failure {
|
||||
persist_input_completion(ctx, result.input_id, &result.input_name, state).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn complete_staged_input<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
|
||||
ctx: &PlaylistProcessingContext<E, M>,
|
||||
input: &tuliprox_core::model::ConfigInput,
|
||||
result: &mut InputDownloadResult,
|
||||
) {
|
||||
let state = result.update_state();
|
||||
if result.input_telemetry.is_some() || state == PlaylistUpdateState::Failure {
|
||||
persist_input_completion(ctx, input.id, &input.name, state).await;
|
||||
// Report the staged input's own facts before its parent consumes them.
|
||||
// Successful in-run reuse has no new acquisition and stays silent.
|
||||
super::ingest::report_input_completion(
|
||||
&ctx.events,
|
||||
&ctx.run_id,
|
||||
ctx.execution_order,
|
||||
input.id,
|
||||
&input.name,
|
||||
state,
|
||||
result.input_telemetry.as_ref(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn persist_input_completion<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
|
||||
ctx: &PlaylistProcessingContext<E, M>,
|
||||
input_id: u16,
|
||||
input_name: &Arc<str>,
|
||||
state: PlaylistUpdateState,
|
||||
) {
|
||||
// Same per-input lock and canonical writer as acquisition. Called after its
|
||||
// own critical section (the staged caller still owns the parent lock).
|
||||
let _input_lock = ctx.get_input_lock(input_name).await;
|
||||
let storage_dir = ctx.config.config.load().storage_dir.clone();
|
||||
let path = resolve_input_storage_path(&storage_dir, input_name).await;
|
||||
{
|
||||
// Shared by source contexts of this run only; never compare with the
|
||||
// previous run's persisted result. Weaker/equal completions do not move
|
||||
// the timestamp belonging to an already recorded stronger result.
|
||||
let mut completions = ctx.input_completions.lock().await;
|
||||
if let Some(current) = completions.get(&input_id) {
|
||||
use PlaylistUpdateState::{Failure, Partial, Success};
|
||||
match (current, state) {
|
||||
(Failure, _) | (Partial, Partial | Success) | (Success, Success) => return,
|
||||
(Partial, Failure) | (Success, Partial | Failure) => {}
|
||||
}
|
||||
}
|
||||
completions.insert(input_id, state);
|
||||
}
|
||||
let mut status = load_input_status(&path);
|
||||
status.last_input_update = Some(PersistedPlaylistUpdateInputResult {
|
||||
state,
|
||||
timestamp: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(),
|
||||
});
|
||||
// Do not touch the cache's cluster status/timestamp or its proven runtime facts.
|
||||
save_input_status(&path, &status);
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
mod persisted_input_status {
|
||||
use super::*;
|
||||
use crate::processor::playlist::input_status::{persist_input_completion, persist_input_job_result};
|
||||
use shared::model::PersistedPlaylistUpdateInputResult;
|
||||
|
||||
fn acquired_result(state: PlaylistUpdateState) -> InputJobResult {
|
||||
let mut result = super::super::playlist_update_run_result(
|
||||
7,
|
||||
InputJobState::Ready,
|
||||
vec![],
|
||||
state == PlaylistUpdateState::Partial,
|
||||
);
|
||||
result.input_telemetry = Some(PlaylistUpdateInputTelemetry {
|
||||
refresh_policy: InputRefreshPolicy::REFRESH,
|
||||
source: Some(PlaylistUpdateDataSource::Provider),
|
||||
clusters: Vec::new(),
|
||||
});
|
||||
result
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_run_precedence_is_isolated_by_stable_input_id() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = processing_context(temp.path());
|
||||
let failed = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false);
|
||||
persist_input_job_result(&ctx, &failed).await;
|
||||
let mut independent = acquired_result(PlaylistUpdateState::Success);
|
||||
independent.input_id = 8;
|
||||
independent.input_name = "independent".intern();
|
||||
persist_input_job_result(&ctx, &independent).await;
|
||||
for (result, state) in [(failed, PlaylistUpdateState::Failure), (independent, PlaylistUpdateState::Success)] {
|
||||
let path =
|
||||
input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await;
|
||||
assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, state);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_playlist_update_status_staged_completion_is_independent_of_parent_failure() {
|
||||
for (parent_fails, staged_fails) in [(false, false), (true, false), (false, true)] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let events = CollectSink::default();
|
||||
let ctx = processing_context_with_events(temp.path(), events.clone());
|
||||
let parent_path = temp.path().join("parent.m3u");
|
||||
let staged_path = temp.path().join("overlay.m3u");
|
||||
for (path, fails) in [(&parent_path, parent_fails), (&staged_path, staged_fails)] {
|
||||
if !fails {
|
||||
std::fs::write(path, "#EXTM3U\n#EXTINF:-1,Fixture\nhttp://stream.example/fixture\n").unwrap();
|
||||
}
|
||||
}
|
||||
let parent = Arc::new(ConfigInput {
|
||||
id: 7,
|
||||
name: "parent".intern(),
|
||||
input_type: InputType::M3u,
|
||||
url: parent_path.to_string_lossy().into_owned(),
|
||||
enabled: true,
|
||||
..ConfigInput::default()
|
||||
});
|
||||
let mut staged = ConfigInput {
|
||||
id: 8,
|
||||
name: "staged".intern(),
|
||||
input_type: InputType::Staged,
|
||||
url: staged_path.to_string_lossy().into_owned(),
|
||||
enabled: true,
|
||||
staged: Some(tuliprox_core::model::ConfigInputStaged {
|
||||
for_input: Some(parent.name.clone()),
|
||||
clusters: ClusterFlags::Live,
|
||||
}),
|
||||
..ConfigInput::default()
|
||||
};
|
||||
staged.resolve_staged_download_type();
|
||||
let staged = Arc::new(staged);
|
||||
ctx.config.sources.store(Arc::new(SourcesConfig {
|
||||
inputs: vec![parent.clone(), staged.clone()],
|
||||
..SourcesConfig::default()
|
||||
}));
|
||||
let result = download_input(&ctx, &parent, false).await;
|
||||
assert_eq!(!result.errors.is_empty(), parent_fails || staged_fails);
|
||||
let path = input_storage_path(&ctx, &staged).await;
|
||||
let persisted = input_cache::load_input_status(&path);
|
||||
assert_eq!(
|
||||
persisted.last_input_update.unwrap().state,
|
||||
if staged_fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success }
|
||||
);
|
||||
assert_eq!(persisted.clusters.len(), 4);
|
||||
assert!(persisted.clusters["default"].last_update.is_none(), "document cache and cluster facts stay separate");
|
||||
for cluster in XTREAM_CLUSTER {
|
||||
assert!(persisted.clusters[cluster.as_ref()].last_update.is_some());
|
||||
}
|
||||
let completions = staged_completion::input_completions(&events, staged.id);
|
||||
assert_eq!(completions.len(), 1, "real staged download must report its own completion");
|
||||
assert_eq!(completions[0].state, persisted.last_input_update.map(|result| result.state));
|
||||
assert_eq!(completions[0].run_id, Some(ctx.run_id.clone()));
|
||||
assert_eq!(completions[0].execution_order, Some(ctx.execution_order));
|
||||
let telemetry = completions[0].input_telemetry.as_ref().unwrap();
|
||||
assert_eq!(telemetry.source, None);
|
||||
assert!(telemetry.clusters.iter().all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider)));
|
||||
assert!(staged_completion::input_completions(&events, parent.id).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_delayed_source_results_keep_strongest_completion_until_next_run() {
|
||||
for weaker in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial] {
|
||||
for failure_first in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let events = CollectSink::default();
|
||||
let ctx = processing_context_with_events(temp.path(), events.clone());
|
||||
let acquired = acquired_result(weaker);
|
||||
let failed_reuse = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false);
|
||||
let (first, delayed) = if failure_first { (failed_reuse, acquired) } else { (acquired, failed_reuse) };
|
||||
let path =
|
||||
input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &first.input_name).await;
|
||||
let mut original = input_cache::InputStatus::default();
|
||||
original.clusters.insert(
|
||||
"live".into(),
|
||||
input_cache::ClusterStatus {
|
||||
status: input_cache::ClusterState::Failed,
|
||||
timestamp: 123,
|
||||
last_update: Some(PersistedPlaylistUpdateClusterSnapshot {
|
||||
source: Some(PlaylistUpdateDataSource::Cache),
|
||||
policy: Some(InputRefreshPolicy::FORCE),
|
||||
quality_guard_threshold: Some(95),
|
||||
..PersistedPlaylistUpdateClusterSnapshot::default()
|
||||
}),
|
||||
},
|
||||
);
|
||||
input_cache::save_input_status(&path, &original);
|
||||
let (release, other_input_finished) = tokio::sync::oneshot::channel();
|
||||
let delayed_ctx = ctx.clone();
|
||||
// Source A already holds its shared-input result, but cannot drain
|
||||
// its sorted result list until another input job has completed.
|
||||
let delayed_source = async move {
|
||||
let source_results = vec![delayed];
|
||||
other_input_finished.await.unwrap();
|
||||
for result in source_results {
|
||||
persist_input_job_result(&delayed_ctx, &result).await;
|
||||
report_input_job_completion(
|
||||
&delayed_ctx.events,
|
||||
&delayed_ctx.run_id,
|
||||
delayed_ctx.execution_order,
|
||||
&result,
|
||||
);
|
||||
}
|
||||
};
|
||||
let first_source = async {
|
||||
persist_input_job_result(&ctx, &first).await;
|
||||
report_input_job_completion(&ctx.events, &ctx.run_id, ctx.execution_order, &first);
|
||||
let mut persisted = input_cache::load_input_status(&path);
|
||||
assert_eq!(persisted.last_input_update.unwrap().state, first.update_state());
|
||||
// Distinct sentinel proves a delayed weaker result does not
|
||||
// advance the stronger completion's timestamp, without sleeps.
|
||||
persisted.last_input_update.as_mut().unwrap().timestamp = 456;
|
||||
input_cache::save_input_status(&path, &persisted);
|
||||
release.send(()).unwrap();
|
||||
};
|
||||
tokio::join!(delayed_source, first_source);
|
||||
let final_status = input_cache::load_input_status(&path);
|
||||
let completion = final_status.last_input_update.unwrap();
|
||||
assert_eq!(
|
||||
completion.state,
|
||||
PlaylistUpdateState::Failure,
|
||||
"weaker={weaker:?}, failure_first={failure_first}"
|
||||
);
|
||||
if failure_first {
|
||||
assert_eq!(completion.timestamp, 456);
|
||||
}
|
||||
assert_eq!(final_status.clusters, original.clusters);
|
||||
let states: Vec<_> = events
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
EventMessage::PlaylistUpdateProgress(progress) => progress.state,
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
states,
|
||||
if failure_first {
|
||||
vec![PlaylistUpdateState::Failure, weaker]
|
||||
} else {
|
||||
vec![weaker, PlaylistUpdateState::Failure]
|
||||
}
|
||||
);
|
||||
|
||||
let mut next = processing_context(temp.path());
|
||||
next.run_id = "next-successful-run".into();
|
||||
next.execution_order = 2.into();
|
||||
let success = acquired_result(PlaylistUpdateState::Success);
|
||||
persist_input_job_result(&next, &success).await;
|
||||
let replacement = input_cache::load_input_status(&path);
|
||||
assert_eq!(replacement.last_input_update.unwrap().state, PlaylistUpdateState::Success);
|
||||
assert_eq!(replacement.clusters, original.clusters);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_parent_download_persists_staged_own_per_cluster_success_and_failure() {
|
||||
for fails in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut responses = fixture_responses([1, 0, 0]);
|
||||
if fails {
|
||||
responses.insert("get_live_streams".into(), "not JSON".into());
|
||||
}
|
||||
let server = TestXtreamServer::start_with_responses(responses);
|
||||
let events = CollectSink::default();
|
||||
let ctx = processing_context_with_events(temp.path(), events.clone());
|
||||
let parent_file = temp.path().join("parent.m3u");
|
||||
std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap();
|
||||
let parent = Arc::new(ConfigInput {
|
||||
id: 7,
|
||||
name: "parent".intern(),
|
||||
input_type: InputType::M3u,
|
||||
url: parent_file.to_string_lossy().into_owned(),
|
||||
enabled: true,
|
||||
..ConfigInput::default()
|
||||
});
|
||||
let mut staged =
|
||||
(*test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, 0, &[]))
|
||||
.clone();
|
||||
staged.id = 8;
|
||||
staged.name = "own-staged".intern();
|
||||
staged.input_type = InputType::Staged;
|
||||
staged.staged_type = shared::model::StagedInputType::Xtream;
|
||||
staged.staged = Some(tuliprox_core::model::ConfigInputStaged {
|
||||
for_input: Some(parent.name.clone()),
|
||||
clusters: ClusterFlags::Live,
|
||||
});
|
||||
staged.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto {
|
||||
skip_vod: true,
|
||||
skip_series: true,
|
||||
update_quality: ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 },
|
||||
..ConfigInputOptionsDto::default()
|
||||
}));
|
||||
staged.resolve_staged_download_type(); // Same resolver used when SourcesConfig is built.
|
||||
let staged = Arc::new(staged);
|
||||
ctx.config.sources.store(Arc::new(SourcesConfig {
|
||||
inputs: vec![parent.clone(), staged.clone()],
|
||||
..SourcesConfig::default()
|
||||
}));
|
||||
let staged_path = input_storage_path(&ctx, &staged).await;
|
||||
// Referencing or directly skipping a staged input is not an acquisition.
|
||||
let skipped = download_input(&ctx, &staged, false).await;
|
||||
assert!(skipped.input_telemetry.is_none());
|
||||
assert!(input_cache::load_input_status(&staged_path).last_input_update.is_none());
|
||||
assert!(staged_completion::input_completions(&events, staged.id).is_empty());
|
||||
|
||||
let result = download_input(&ctx, &parent, false).await;
|
||||
let status = input_cache::load_input_status(&staged_path);
|
||||
assert!(!status.clusters.contains_key("default"));
|
||||
assert_eq!(status.clusters.len(), 1);
|
||||
let completion =
|
||||
status.last_input_update.expect("actual indirect staged acquisition has its own completion");
|
||||
assert_eq!(
|
||||
completion.state,
|
||||
if fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success }
|
||||
);
|
||||
assert!(completion.timestamp > 0);
|
||||
let completions = staged_completion::input_completions(&events, staged.id);
|
||||
assert_eq!(completions.len(), 1);
|
||||
assert_eq!(completions[0].state, Some(completion.state));
|
||||
assert_eq!(completions[0].run_id, Some(ctx.run_id.clone()));
|
||||
assert_eq!(completions[0].execution_order, Some(ctx.execution_order));
|
||||
let snapshot = status.clusters["live"].last_update.unwrap();
|
||||
assert_eq!(snapshot.source, Some(PlaylistUpdateDataSource::Provider));
|
||||
assert_eq!(snapshot.policy, None);
|
||||
if fails {
|
||||
assert_eq!(snapshot.quality_guard_threshold, Some(95));
|
||||
assert_eq!(snapshot.quality, None);
|
||||
} else {
|
||||
assert_eq!(snapshot.quality_guard_threshold, None, "the evaluation already owns its threshold");
|
||||
let quality = snapshot.quality.unwrap();
|
||||
assert_eq!(quality.threshold, 95);
|
||||
assert_eq!(quality.decision, PersistedPlaylistUpdateQualityDecision::Accepted);
|
||||
assert_eq!(quality.achieved_quality, None, "bootstrap has no invented Quality");
|
||||
}
|
||||
assert_eq!(result.errors.is_empty(), !fails);
|
||||
let parent_path = input_storage_path(&ctx, &parent).await;
|
||||
assert_ne!(parent_path, staged_path);
|
||||
assert!(
|
||||
input_cache::load_input_status(&parent_path).last_input_update.is_none(),
|
||||
"staged result does not invent a parent job completion"
|
||||
);
|
||||
if !fails {
|
||||
let bytes = std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap();
|
||||
let reuse = download_input(&ctx, &parent, false).await;
|
||||
assert!(reuse.input_telemetry.is_none());
|
||||
let mut staged_reuse = download_input(&ctx, &staged, true).await;
|
||||
assert!(staged_reuse.input_telemetry.is_none());
|
||||
super::super::input_status::complete_staged_input(&ctx, &staged, &mut staged_reuse).await;
|
||||
assert_eq!(std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap(), bytes);
|
||||
assert_eq!(staged_completion::input_completions(&events, staged.id), completions);
|
||||
}
|
||||
let requests = server.finish();
|
||||
assert_eq!(
|
||||
requests.iter().filter(|action| action.as_str() == "get_live_streams").count(),
|
||||
1,
|
||||
"no extra staged acquisition on parent reuse"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_completion_replaces_one_result_without_touching_cache_or_cluster_snapshots() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = processing_context(temp.path());
|
||||
let input_name: Arc<str> = "result-input".intern();
|
||||
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input_name).await;
|
||||
let mut original = input_cache::InputStatus::default();
|
||||
input_cache::update_cluster_status(&mut original, "default", input_cache::ClusterState::Ok);
|
||||
original.clusters.get_mut("default").unwrap().timestamp = 1; // Expired cache must stay expired.
|
||||
input_cache::replace_cluster_snapshot(
|
||||
&mut original,
|
||||
"default",
|
||||
PersistedPlaylistUpdateClusterSnapshot {
|
||||
source: Some(PlaylistUpdateDataSource::Cache),
|
||||
..PersistedPlaylistUpdateClusterSnapshot::default()
|
||||
},
|
||||
);
|
||||
input_cache::save_input_status(&path, &original);
|
||||
for state in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial, PlaylistUpdateState::Failure] {
|
||||
persist_input_completion(&ctx, 7, &input_name, state).await;
|
||||
let reloaded = input_cache::load_input_status(&path);
|
||||
let result = reloaded.last_input_update.unwrap();
|
||||
assert_eq!(result.state, state);
|
||||
assert!(result.timestamp > 1);
|
||||
assert_eq!(reloaded.clusters, original.clusters);
|
||||
assert!(!input_cache::is_cache_valid(&reloaded, "default", 3600));
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap()).unwrap();
|
||||
assert!(json["last_input_update"].is_object());
|
||||
assert_eq!(json.as_object().unwrap().len(), 2, "only clusters + one result, no history");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_in_run_reuse_preserves_original_partial_result_and_timestamp() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = processing_context(temp.path());
|
||||
let mut result = super::super::playlist_update_run_result(7, InputJobState::Ready, vec![], true);
|
||||
result.input_telemetry = Some(PlaylistUpdateInputTelemetry {
|
||||
refresh_policy: InputRefreshPolicy::NORMAL,
|
||||
source: Some(PlaylistUpdateDataSource::Provider),
|
||||
clusters: Vec::new(),
|
||||
});
|
||||
persist_input_job_result(&ctx, &result).await;
|
||||
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await;
|
||||
let mut original = input_cache::load_input_status(&path);
|
||||
assert_eq!(original.last_input_update.unwrap().state, PlaylistUpdateState::Partial);
|
||||
original.last_input_update.as_mut().unwrap().timestamp = 123;
|
||||
input_cache::save_input_status(&path, &original);
|
||||
let before = std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap();
|
||||
result.had_quality_rejections = false;
|
||||
result.input_telemetry = None; // Same existing absence of acquisition facts as a real in-run reuse.
|
||||
persist_input_job_result(&ctx, &result).await;
|
||||
assert_eq!(std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap(), before);
|
||||
result.state = InputJobState::Failed; // A genuine technical completion must still replace the prior result.
|
||||
persist_input_job_result(&ctx, &result).await;
|
||||
assert_eq!(
|
||||
input_cache::load_input_status(&path).last_input_update.unwrap().state,
|
||||
PlaylistUpdateState::Failure
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_completion_covers_every_input_type_without_artificial_clusters() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = processing_context(temp.path());
|
||||
for (id, input_type) in [
|
||||
InputType::Xtream,
|
||||
InputType::XtreamBatch,
|
||||
InputType::Stalker,
|
||||
InputType::StalkerBatch,
|
||||
InputType::M3u,
|
||||
InputType::M3uBatch,
|
||||
InputType::Library,
|
||||
InputType::Plex,
|
||||
InputType::Emby,
|
||||
InputType::Jellyfin,
|
||||
InputType::Staged,
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let input = ConfigInput {
|
||||
id: u16::try_from(id).unwrap(),
|
||||
input_type,
|
||||
name: format!("input-{id}").intern(),
|
||||
..ConfigInput::default()
|
||||
};
|
||||
let result = panicked_input_job(id, &input);
|
||||
persist_input_job_result(&ctx, &result).await;
|
||||
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await;
|
||||
let persisted = input_cache::load_input_status(&path);
|
||||
assert!(persisted.clusters.is_empty());
|
||||
assert!(
|
||||
matches!(persisted.last_input_update, Some(PersistedPlaylistUpdateInputResult {state: PlaylistUpdateState::Failure, timestamp}) if timestamp > 0)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
//! M3U's document adapter at the existing locked input acquisition boundary.
|
||||
use super::ingest::{cluster_is_configured, PIPELINE_TRANSPARENCY_CLUSTERS};
|
||||
use shared::model::{M3uPlaylistItem, UpdateQualityPolicy};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
};
|
||||
use tuliprox_core::model::{evaluate_update_quality, AppConfig, ClusterForceUpdate, ConfigInput};
|
||||
use tuliprox_iptv::provider::PlaylistFetch;
|
||||
use tuliprox_repository::{get_input_m3u_playlist_file_path, get_input_storage_path, load_input_m3u_playlist};
|
||||
|
||||
/// Called while the existing per-input lock protects baseline, decision and subsequent persistence.
|
||||
pub(super) async fn effective_m3u_fetch(
|
||||
config: &Arc<AppConfig>,
|
||||
input: &ConfigInput,
|
||||
policy: UpdateQualityPolicy,
|
||||
mut fetch: PlaylistFetch,
|
||||
) -> PlaylistFetch {
|
||||
if !fetch.is_ok() {
|
||||
return fetch;
|
||||
}
|
||||
// Match the existing M3U writer's last-wins primary key population. This also
|
||||
// keeps the target source and raw group catalogs consistent with its BTree.
|
||||
let mut winners = HashMap::new();
|
||||
for (group_index, group) in fetch.groups.iter_mut().enumerate() {
|
||||
for item in std::mem::take(&mut group.channels) {
|
||||
winners.insert(M3uPlaylistItem::from(&item).provider_id, (group_index, item));
|
||||
}
|
||||
}
|
||||
let mut winners: Vec<_> = winners.into_values().collect();
|
||||
winners.sort_by_key(|(group, item)| (*group, item.header.source_ordinal));
|
||||
for (group_index, item) in winners {
|
||||
fetch.groups[group_index].channels.push(item);
|
||||
}
|
||||
fetch.groups.retain(|group| !group.channels.is_empty());
|
||||
|
||||
if policy == UpdateQualityPolicy::Enforce
|
||||
&& input.options.as_ref().is_none_or(|options| {
|
||||
PIPELINE_TRANSPARENCY_CLUSTERS.into_iter().all(|cluster| options.update_quality.threshold(cluster) == 0)
|
||||
})
|
||||
{
|
||||
return fetch;
|
||||
}
|
||||
|
||||
let needs_previous = PIPELINE_TRANSPARENCY_CLUSTERS.into_iter().any(|cluster| {
|
||||
!cluster_is_configured(input, cluster)
|
||||
|| (policy == UpdateQualityPolicy::Enforce
|
||||
&& input.options.as_ref().is_some_and(|options| options.update_quality.threshold(cluster) > 0))
|
||||
});
|
||||
let previous = if needs_previous {
|
||||
let storage_dir = config.config.load().storage_dir.clone();
|
||||
let loaded = async {
|
||||
let storage = get_input_storage_path(&input.name, &storage_dir).await.map_err(|error| {
|
||||
shared::error::TuliproxError::RepositoryM3u(format!("Cannot read M3U baseline: {error}"))
|
||||
})?;
|
||||
load_input_m3u_playlist(config, &get_input_m3u_playlist_file_path(&storage, &input.name)).await
|
||||
}
|
||||
.await;
|
||||
match loaded {
|
||||
Ok(groups) => groups,
|
||||
Err(error) => return PlaylistFetch::failed(error),
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut retained = Vec::new();
|
||||
for cluster in PIPELINE_TRANSPARENCY_CLUSTERS {
|
||||
if !cluster_is_configured(input, cluster) {
|
||||
retained.push(cluster);
|
||||
continue;
|
||||
}
|
||||
let threshold = input.options.as_ref().map_or(0, |options| options.update_quality.threshold(cluster));
|
||||
let candidate_count =
|
||||
fetch.groups.iter().filter(|group| group.xtream_cluster == cluster).map(|group| group.channels.len()).sum();
|
||||
let current_count = previous
|
||||
.iter()
|
||||
.filter(|group| group.xtream_cluster == cluster)
|
||||
.map(|group| group.channels.len())
|
||||
.reduce(|total, count| total + count);
|
||||
if policy == UpdateQualityPolicy::Bypass {
|
||||
fetch.force_updates.push(ClusterForceUpdate { cluster, candidate_count, configured_threshold: threshold });
|
||||
continue;
|
||||
}
|
||||
let decision = evaluate_update_quality(current_count, candidate_count, threshold);
|
||||
if let Some(rejection) = decision.rejection(cluster) {
|
||||
fetch.quality_rejections.push(rejection);
|
||||
retained.push(cluster);
|
||||
} else if let Some(acceptance) = decision.acceptance(cluster) {
|
||||
fetch.quality_acceptances.push(acceptance);
|
||||
}
|
||||
}
|
||||
// Quality decisions must not reorder accepted candidate groups. Append retained
|
||||
// groups in their existing persisted order, without regrouping either source.
|
||||
fetch.groups.retain(|group| !retained.contains(&group.xtream_cluster));
|
||||
fetch.groups.extend(previous.into_iter().filter(|group| retained.contains(&group.xtream_cluster)));
|
||||
// Retention can combine an old cluster with a newly reclassified primary key.
|
||||
// The existing document BTree cannot represent both; fail before publication
|
||||
// instead of losing a retained row or exposing different target/catalog facts.
|
||||
let mut keys = HashSet::new();
|
||||
if fetch
|
||||
.groups
|
||||
.iter()
|
||||
.flat_map(|group| &group.channels)
|
||||
.any(|item| !keys.insert(M3uPlaylistItem::from(item).provider_id))
|
||||
{
|
||||
return PlaylistFetch::failed(shared::error::TuliproxError::RepositoryM3u(
|
||||
"Conflicting M3U keys across retained and candidate clusters".to_owned(),
|
||||
));
|
||||
}
|
||||
fetch
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
mod m3u_update_quality {
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
use tuliprox_repository::{load_input_m3u_playlist, raw_group_catalog_path, RawInputGroupCatalog};
|
||||
|
||||
fn document(prefix: &str, counts: [usize; 3]) -> String {
|
||||
let mut text = String::from("#EXTM3U\n");
|
||||
for (cluster, count) in XTREAM_CLUSTER.into_iter().zip(counts) {
|
||||
for index in 0..count {
|
||||
let (kind, group, title) = match cluster {
|
||||
XtreamCluster::Live => ("live", format!("{prefix}-live"), format!("Channel {index}")),
|
||||
XtreamCluster::Video => ("movie", format!("{prefix}-vod"), format!("Movie {index}")),
|
||||
XtreamCluster::Series => {
|
||||
("series", format!("{prefix}-show-{index}"), format!("Show {index} S01E01"))
|
||||
}
|
||||
};
|
||||
let id = match cluster {
|
||||
XtreamCluster::Live => 1000,
|
||||
XtreamCluster::Video => 2000,
|
||||
XtreamCluster::Series => 3000,
|
||||
} + index;
|
||||
writeln!(text, "#EXTGRP:{prefix}-{cluster}\n#EXTINF:-1 xui-id=\"{id}\" tvg-type=\"{kind}\" group-title=\"{group}\",{title}\nhttp://stream.example/{prefix}/{cluster}/{id}").unwrap();
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
async fn fixture(
|
||||
root: &Path,
|
||||
previous: Option<[usize; 3]>,
|
||||
thresholds: [u8; 3],
|
||||
) -> (PlaylistProcessingContext<shared::model::NoopSink>, Arc<ConfigInput>) {
|
||||
let ctx = processing_context(root);
|
||||
let input = Arc::new(ConfigInput {
|
||||
id: 27,
|
||||
name: "m3u-quality".intern(),
|
||||
input_type: InputType::M3u,
|
||||
enabled: true,
|
||||
url: root.join("provider.m3u").to_string_lossy().into_owned(),
|
||||
options: Some(ConfigInputOptions::from(&ConfigInputOptionsDto {
|
||||
update_quality: ConfigInputUpdateQualityDto {
|
||||
live: thresholds[0],
|
||||
vod: thresholds[1],
|
||||
series: thresholds[2],
|
||||
},
|
||||
..ConfigInputOptionsDto::default()
|
||||
})),
|
||||
..ConfigInput::default()
|
||||
});
|
||||
if let Some(counts) = previous {
|
||||
tokio::fs::write(&input.url, document("previous", counts)).await.unwrap();
|
||||
let (groups, errors) =
|
||||
tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), &input)
|
||||
.await;
|
||||
assert!(errors.is_empty());
|
||||
let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, groups).await;
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
}
|
||||
(ctx, input)
|
||||
}
|
||||
|
||||
async fn run(
|
||||
ctx: &PlaylistProcessingContext<shared::model::NoopSink>,
|
||||
input: &Arc<ConfigInput>,
|
||||
counts: [usize; 3],
|
||||
) -> InputDownloadResult {
|
||||
tokio::fs::write(&input.url, document("candidate", counts)).await.unwrap();
|
||||
download_input(ctx, input, false).await
|
||||
}
|
||||
|
||||
fn counts(groups: &[PlaylistGroup]) -> [usize; 3] {
|
||||
XTREAM_CLUSTER
|
||||
.map(|cluster| groups.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.channels.len()).sum())
|
||||
}
|
||||
|
||||
async fn verify_effective(
|
||||
ctx: &PlaylistProcessingContext<shared::model::NoopSink>,
|
||||
input: &ConfigInput,
|
||||
result: &mut InputDownloadResult,
|
||||
expected: [usize; 3],
|
||||
) {
|
||||
assert!(result.errors.is_empty(), "{:?}", result.errors);
|
||||
assert!(result.storage_error.is_none(), "{:?}", result.storage_error);
|
||||
assert_eq!(result.job_state(), InputJobState::Ready);
|
||||
let effective = result.source.take_groups();
|
||||
assert_eq!(counts(&effective), expected);
|
||||
let storage = tuliprox_repository::get_input_storage_path(&input.name, &ctx.config.config.load().storage_dir)
|
||||
.await
|
||||
.unwrap();
|
||||
let saved = load_input_m3u_playlist(
|
||||
&ctx.config,
|
||||
&tuliprox_repository::get_input_m3u_playlist_file_path(&storage, &input.name),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(counts(&saved), expected);
|
||||
for cluster in XTREAM_CLUSTER {
|
||||
let catalog: RawInputGroupCatalog =
|
||||
serde_json::from_slice(&tokio::fs::read(raw_group_catalog_path(&storage, cluster)).await.unwrap())
|
||||
.unwrap();
|
||||
let mut expected_groups: Vec<_> =
|
||||
effective.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.title.to_string()).collect();
|
||||
expected_groups.sort();
|
||||
expected_groups.dedup();
|
||||
assert_eq!(catalog.groups, expected_groups);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_all_clusters_accepted() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
let mut result = run(&ctx, &input, [9, 11, 10]).await;
|
||||
assert!(result.quality_rejections.is_empty());
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
|
||||
verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await;
|
||||
}
|
||||
|
||||
async fn order_candidate(
|
||||
ctx: &PlaylistProcessingContext<shared::model::NoopSink>,
|
||||
input: &ConfigInput,
|
||||
prefix: &str,
|
||||
) -> tuliprox_iptv::provider::PlaylistFetch {
|
||||
let mut body = String::from("#EXTM3U\n");
|
||||
for (id, kind, group) in [
|
||||
(2000, "movie", "obsolete"),
|
||||
(2000, "movie", "movies-a"),
|
||||
(1000, "live", "live-a"),
|
||||
(2001, "movie", "movies-b"),
|
||||
(1001, "live", "live-b"),
|
||||
(3000, "series", "shows-a"),
|
||||
(3001, "series", "shows-b"),
|
||||
] {
|
||||
writeln!(body, "#EXTGRP:{prefix}-{group}\n#EXTINF:-1 tvg-type=\"{kind}\" group-title=\"{prefix}-{group}\",Item {id} S01E01\nhttp://stream.example/{kind}/{id}").unwrap();
|
||||
}
|
||||
tokio::fs::write(&input.url, body).await.unwrap();
|
||||
let (groups, errors) =
|
||||
tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), input).await;
|
||||
assert!(errors.is_empty());
|
||||
assert_eq!(
|
||||
group_names(&groups),
|
||||
["obsolete", "movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"]
|
||||
.map(|name| format!("{prefix}-{name}"))
|
||||
);
|
||||
tuliprox_iptv::provider::PlaylistFetch::groups(groups)
|
||||
}
|
||||
|
||||
fn group_names(groups: &[PlaylistGroup]) -> Vec<String> {
|
||||
groups.iter().map(|group| group.title.to_string()).collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_preserves_candidate_group_order_when_disabled() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await;
|
||||
let candidate = order_candidate(&ctx, &input, "candidate").await;
|
||||
let effective = super::super::super::m3u_quality::effective_m3u_fetch(
|
||||
&ctx.config,
|
||||
&input,
|
||||
shared::model::UpdateQualityPolicy::Enforce,
|
||||
candidate,
|
||||
)
|
||||
.await;
|
||||
assert!(effective.errors.is_empty());
|
||||
assert!(effective.quality_acceptances.is_empty() && effective.quality_rejections.is_empty());
|
||||
assert_eq!(
|
||||
group_names(&effective.groups),
|
||||
["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"]
|
||||
.map(|name| format!("candidate-{name}"))
|
||||
);
|
||||
assert_eq!(counts(&effective.groups), [2, 2, 2], "disabled quality processing must collapse duplicate keys");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_preserves_candidate_group_order_when_all_clusters_accepted() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([2, 2, 2]), [100, 100, 100]).await;
|
||||
let candidate = order_candidate(&ctx, &input, "candidate").await;
|
||||
let effective = super::super::super::m3u_quality::effective_m3u_fetch(
|
||||
&ctx.config,
|
||||
&input,
|
||||
shared::model::UpdateQualityPolicy::Enforce,
|
||||
candidate,
|
||||
)
|
||||
.await;
|
||||
assert!(effective.errors.is_empty() && effective.quality_rejections.is_empty());
|
||||
assert_eq!(effective.quality_acceptances.len(), 3);
|
||||
assert_eq!(
|
||||
group_names(&effective.groups),
|
||||
["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"].map(|name| format!("candidate-{name}"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_mixed_fallback_preserves_relative_group_order() {
|
||||
for _ in 0..3 {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), None, [100, 100, 100]).await;
|
||||
let mut previous = order_candidate(&ctx, &input, "previous").await.groups;
|
||||
previous.retain(|group| group.title.as_ref() != "previous-obsolete");
|
||||
let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, previous).await;
|
||||
assert!(error.is_none());
|
||||
let mut previous = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
let previous = previous.take_groups();
|
||||
assert_eq!(
|
||||
previous
|
||||
.iter()
|
||||
.filter(|group| group.xtream_cluster == XtreamCluster::Series)
|
||||
.map(|group| group.title.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
["previous-shows-a", "previous-shows-b"]
|
||||
);
|
||||
let mut candidate = order_candidate(&ctx, &input, "candidate").await;
|
||||
candidate.groups.retain(|group| group.title.as_ref() != "candidate-shows-b");
|
||||
let effective = super::super::super::m3u_quality::effective_m3u_fetch(
|
||||
&ctx.config,
|
||||
&input,
|
||||
shared::model::UpdateQualityPolicy::Enforce,
|
||||
candidate,
|
||||
)
|
||||
.await;
|
||||
assert!(effective.errors.is_empty());
|
||||
assert_eq!(effective.quality_rejections.len(), 1);
|
||||
assert_eq!(effective.quality_rejections[0].cluster, XtreamCluster::Series);
|
||||
assert_eq!(
|
||||
group_names(&effective.groups),
|
||||
[
|
||||
"candidate-movies-a",
|
||||
"candidate-live-a",
|
||||
"candidate-movies-b",
|
||||
"candidate-live-b",
|
||||
"previous-shows-a",
|
||||
"previous-shows-b"
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_retains_only_rejected_cluster() {
|
||||
for disk in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
config.disk_based_processing = disk;
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
let mut result = run(&ctx, &input, [9, 11, 8]).await;
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Partial);
|
||||
assert_eq!(result.quality_rejections.len(), 1);
|
||||
assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series);
|
||||
verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_zero_candidate_retains_previous_cluster() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
let mut result = run(&ctx, &input, [10, 10, 0]).await;
|
||||
assert_eq!(result.quality_rejections[0].candidate_count, 0);
|
||||
verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_disabled() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [0, 0, 0]).await;
|
||||
let mut result = run(&ctx, &input, [1, 1, 0]).await;
|
||||
assert!(result.quality_rejections.is_empty());
|
||||
assert!(result
|
||||
.input_telemetry
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.clusters
|
||||
.iter()
|
||||
.all(|c| c.quality.is_none() && c.threshold.is_none()));
|
||||
verify_effective(&ctx, &input, &mut result, [1, 1, 0]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_first_import() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), None, [90, 90, 90]).await;
|
||||
let mut result = run(&ctx, &input, [1, 1, 1]).await;
|
||||
assert!(result.quality_rejections.is_empty());
|
||||
assert!(result.input_telemetry.as_ref().unwrap().clusters.iter().all(|c| c.quality.is_none()
|
||||
&& c.baseline_count.is_none()
|
||||
&& c.decision == Some(PlaylistUpdateClusterDecision::Accepted)));
|
||||
verify_effective(&ctx, &input, &mut result, [1, 1, 1]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_download_or_parse_failure() {
|
||||
for body in [None, Some("<html>provider unavailable</html>"), Some("#EXTM3U\n#EXTINF:-1,Truncated\n")] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
if let Some(body) = body {
|
||||
tokio::fs::write(&input.url, body).await.unwrap();
|
||||
} else {
|
||||
tokio::fs::remove_file(&input.url).await.unwrap();
|
||||
}
|
||||
let mut result = download_input(&ctx, &input, false).await;
|
||||
assert!(!result.errors.is_empty());
|
||||
assert!(result.quality_rejections.is_empty());
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
|
||||
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert_eq!(counts(&saved.take_groups()), [10, 10, 10]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_empty_bootstrap_and_structural_empty_use_existing_domain() {
|
||||
for previous in [None, Some([10, 10, 0])] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), previous, [90, 90, 90]).await;
|
||||
let mut result = run(&ctx, &input, if previous.is_some() { [10, 10, 0] } else { [0, 0, 0] }).await;
|
||||
if previous.is_none() {
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
|
||||
assert_eq!(result.quality_rejections.len(), 3);
|
||||
} else {
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Partial);
|
||||
assert_eq!(result.quality_rejections.len(), 1);
|
||||
assert_eq!(result.quality_rejections[0].current_count, 0);
|
||||
assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_skip_preserves_previous_and_reuse_emits_no_new_facts() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
Arc::make_mut(&mut input).options.as_mut().unwrap().flags.set(ConfigInputFlags::SkipSeries);
|
||||
let mut result = run(&ctx, &input, [10, 10, 0]).await;
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
|
||||
assert!(result.quality_rejections.is_empty());
|
||||
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert_eq!(counts(&result.source.take_groups()), [10, 10, 0]);
|
||||
assert_eq!(counts(&saved.take_groups()), [10, 10, 10]);
|
||||
let telemetry = result.input_telemetry.unwrap();
|
||||
assert!(!telemetry.clusters.iter().find(|c| c.cluster == XtreamCluster::Series).unwrap().requested);
|
||||
tokio::fs::remove_file(&input.url).await.unwrap();
|
||||
let reused = download_input(&ctx, &input, false).await;
|
||||
assert!(reused.errors.is_empty());
|
||||
assert!(reused.input_telemetry.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_force_bypasses_guard_and_cache_without_inventing_quality() {
|
||||
for threshold in [0, 95] {
|
||||
for policy in [InputRefreshPolicy::REFRESH, InputRefreshPolicy::FORCE] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (mut ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [threshold; 3]).await;
|
||||
Arc::make_mut(&mut input).cache_duration_seconds = 3_600;
|
||||
let storage = input_storage_path(&ctx, &input).await;
|
||||
let mut status = input_cache::InputStatus::default();
|
||||
input_cache::update_cluster_status(&mut status, "default", input_cache::ClusterState::Ok);
|
||||
input_cache::save_input_status(&storage, &status);
|
||||
set_refresh_policy(&mut ctx, &input, policy);
|
||||
let mut result = run(&ctx, &input, [1, 1, 1]).await;
|
||||
let rejected = threshold > 0 && policy == InputRefreshPolicy::REFRESH;
|
||||
assert_eq!(
|
||||
result.update_state(),
|
||||
if rejected { PlaylistUpdateState::Partial } else { PlaylistUpdateState::Success }
|
||||
);
|
||||
assert_eq!(counts(&result.source.take_groups()), if rejected { [10; 3] } else { [1; 3] });
|
||||
let telemetry = result.input_telemetry.as_ref().unwrap();
|
||||
assert_eq!(telemetry.refresh_policy, policy);
|
||||
assert!(telemetry
|
||||
.clusters
|
||||
.iter()
|
||||
.all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider)));
|
||||
let saved = input_cache::load_input_status(&storage);
|
||||
for cluster in &telemetry.clusters {
|
||||
if policy == InputRefreshPolicy::FORCE {
|
||||
assert_eq!(cluster.decision, Some(PlaylistUpdateClusterDecision::Accepted));
|
||||
assert_eq!((cluster.baseline_count, cluster.quality), (None, None));
|
||||
assert_eq!((cluster.candidate_count, cluster.active_count), (Some(1), Some(1)));
|
||||
let snapshot = saved.clusters[cluster.cluster.as_ref()].last_update.unwrap();
|
||||
assert_eq!(snapshot.quality, None);
|
||||
assert_eq!(snapshot.quality_guard_threshold, Some(threshold));
|
||||
assert_eq!(snapshot.policy, Some(policy));
|
||||
}
|
||||
}
|
||||
let reused = download_input(&ctx, &input, false).await;
|
||||
assert!(reused.input_telemetry.is_none(), "in-run reuse must not create another Force acquisition");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_force_keeps_parse_and_empty_publication_errors() {
|
||||
for body in ["#EXTM3U\n", "#EXTM3U\n#EXTINF:-1,Truncated\n"] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (mut ctx, input) = fixture(temp.path(), Some([10; 3]), [95; 3]).await;
|
||||
set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE);
|
||||
tokio::fs::write(&input.url, body).await.unwrap();
|
||||
let mut result = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
|
||||
assert!(result.quality_rejections.is_empty());
|
||||
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert_eq!(counts(&saved.take_groups()), [10; 3]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_target_processing_uses_retained_and_accepted_groups() {
|
||||
for policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::FORCE] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (mut ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
set_refresh_policy(&mut ctx, &input, policy);
|
||||
tokio::fs::write(&input.url, document("candidate", [10, 0, 10])).await.unwrap();
|
||||
let output = temp.path().join("client.m3u");
|
||||
let mut target = ConfigTarget::from(&ConfigTargetDto {
|
||||
id: 1,
|
||||
name: "m3u-target".into(),
|
||||
enabled: true,
|
||||
..ConfigTargetDto::default()
|
||||
});
|
||||
target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into();
|
||||
target.output = vec![TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput {
|
||||
filename: Some(output.to_string_lossy().into_owned()),
|
||||
include_type_in_url: false,
|
||||
mask_redirect_url: false,
|
||||
filter: None,
|
||||
})];
|
||||
ctx.config.sources.store(Arc::new(SourcesConfig {
|
||||
inputs: vec![input.clone()],
|
||||
sources: vec![ConfigSource { inputs: vec![input.name.clone()], targets: vec![Arc::new(target)] }],
|
||||
..SourcesConfig::default()
|
||||
}));
|
||||
let (_, targets, errors) = process_source(0, Arc::new(ctx)).await;
|
||||
assert!(errors.is_empty(), "{errors:?}");
|
||||
assert_eq!(targets.len(), 1);
|
||||
let published = tokio::fs::read_to_string(output).await.unwrap();
|
||||
assert!(published.contains("/candidate/live/") && published.contains("/candidate/series/"), "{published}");
|
||||
assert_eq!(published.contains("/previous/video/"), policy == InputRefreshPolicy::NORMAL, "{published}");
|
||||
assert!(!published.contains("/previous/live/") && !published.contains("/candidate/video/"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_raw_group_catalog_uses_effective_state() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await;
|
||||
let mut result = run(&ctx, &input, [10, 10, 0]).await;
|
||||
let storage = input_cache::resolve_input_storage_path(&ctx.config.config.load().storage_dir, &input.name).await;
|
||||
let status = input_cache::load_input_status(&storage);
|
||||
for cluster in XTREAM_CLUSTER {
|
||||
let saved = &status.clusters[cluster.as_ref()];
|
||||
let rejected = cluster == XtreamCluster::Series;
|
||||
assert_eq!(
|
||||
saved.status,
|
||||
if rejected { input_cache::ClusterState::Failed } else { input_cache::ClusterState::Ok }
|
||||
);
|
||||
let snapshot = saved.last_update.unwrap();
|
||||
assert_eq!(
|
||||
snapshot.quality.unwrap().decision,
|
||||
if rejected {
|
||||
PersistedPlaylistUpdateQualityDecision::Rejected
|
||||
} else {
|
||||
PersistedPlaylistUpdateQualityDecision::Accepted
|
||||
}
|
||||
);
|
||||
assert_eq!(snapshot.active_count, Some(10));
|
||||
}
|
||||
assert_eq!(status.clusters["default"].status, input_cache::ClusterState::Failed);
|
||||
let series: RawInputGroupCatalog = serde_json::from_slice(
|
||||
&tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Series)).await.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(series.groups, ["previous-series"]);
|
||||
verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [100, 100, 0]).await;
|
||||
let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]);
|
||||
tokio::fs::write(&input.url, candidate).await.unwrap();
|
||||
let mut result = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
|
||||
assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last")));
|
||||
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert_eq!(counts(&saved.take_groups()), [1, 1, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population_when_disabled() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await;
|
||||
let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]);
|
||||
tokio::fs::write(&input.url, candidate).await.unwrap();
|
||||
let mut result = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Success);
|
||||
assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last")));
|
||||
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert_eq!(counts(&saved.take_groups()), [1, 1, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn m3u_update_quality_conflicting_retained_key_cannot_diverge_from_storage() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [90, 0, 0]).await;
|
||||
let candidate = document("candidate", [0, 1, 0]).replace("2000", "1000");
|
||||
tokio::fs::write(&input.url, candidate).await.unwrap();
|
||||
let mut result = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
|
||||
assert!(!result.errors.is_empty());
|
||||
let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert_eq!(counts(&saved.take_groups()), [1, 1, 0]);
|
||||
let storage = input_storage_path(&ctx, &input).await;
|
||||
let catalog: RawInputGroupCatalog = serde_json::from_slice(
|
||||
&tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Live)).await.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(catalog.groups, ["previous-live"]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
use super::{process_sources, MetadataUpdateSink, PlaylistProcessingContext};
|
||||
use shared::{
|
||||
error::TuliproxError,
|
||||
model::{
|
||||
EventMessage, EventSink, LibraryScanResult, PlaylistUpdateProgressDetail, PlaylistUpdateProgressEvent,
|
||||
PlaylistUpdateState, SourceStats,
|
||||
},
|
||||
};
|
||||
use std::{future::Future, io};
|
||||
use tuliprox_core::model::UpdateGuard;
|
||||
use tuliprox_library::library::LibraryProcessor;
|
||||
|
||||
/// Bulk and scheduled runs read the existing catalog. Only an explicit card request scans.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) enum LibraryUpdateMode {
|
||||
#[default]
|
||||
ExistingCatalog,
|
||||
Rescan {
|
||||
input_id: u16,
|
||||
},
|
||||
}
|
||||
|
||||
impl LibraryUpdateMode {
|
||||
pub(crate) fn reloads_input(self, input_id: u16) -> bool { self == Self::Rescan { input_id } }
|
||||
}
|
||||
|
||||
/// The rebuild is not even started until the existing scanner reports complete success.
|
||||
async fn rebuild_after_scan<T>(
|
||||
scan: impl Future<Output = io::Result<LibraryScanResult>>,
|
||||
report: impl FnOnce(LibraryScanResult),
|
||||
rebuild: impl FnOnce() -> T,
|
||||
) -> io::Result<T> {
|
||||
let result = scan.await?;
|
||||
let complete = result.errors == 0;
|
||||
report(result);
|
||||
if !complete {
|
||||
return Err(io::Error::other("Library rescan reported incomplete input data"));
|
||||
}
|
||||
Ok(rebuild())
|
||||
}
|
||||
|
||||
pub(super) async fn process_manual_update<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
|
||||
ctx: &PlaylistProcessingContext<E, M>,
|
||||
update_guard: Option<&UpdateGuard>,
|
||||
) -> (Vec<SourceStats>, Vec<TuliproxError>) {
|
||||
let LibraryUpdateMode::Rescan { input_id } = ctx.library_update_mode else {
|
||||
return process_sources(ctx).await;
|
||||
};
|
||||
let progress = |message: &str, detail| {
|
||||
ctx.events.emit(EventMessage::PlaylistUpdateProgress(
|
||||
PlaylistUpdateProgressEvent::for_run_input(
|
||||
ctx.run_id.clone(),
|
||||
ctx.execution_order,
|
||||
input_id,
|
||||
"Library rescan",
|
||||
message,
|
||||
)
|
||||
.with_detail(detail),
|
||||
));
|
||||
};
|
||||
progress(
|
||||
"Library rescan started; selected target rebuild is waiting",
|
||||
PlaylistUpdateProgressDetail::LibraryRescanStarted,
|
||||
);
|
||||
let result = async {
|
||||
// The worker already owns the playlist permit. Never wait for a second lock:
|
||||
// a concurrent standalone scan makes this request fail, without a lock cycle.
|
||||
let _library_permit = update_guard
|
||||
.and_then(UpdateGuard::try_library)
|
||||
.ok_or_else(|| io::Error::other("Library scan is already running or its guard is unavailable"))?;
|
||||
let (library, metadata, storage_dir) = {
|
||||
let config = ctx.config.config.load();
|
||||
let library = config
|
||||
.library
|
||||
.as_ref()
|
||||
.filter(|library| library.enabled)
|
||||
.ok_or_else(|| io::Error::other("Library is not enabled"))?
|
||||
.clone();
|
||||
(library, config.metadata_update.clone(), config.storage_dir.clone())
|
||||
};
|
||||
let processor = LibraryProcessor::new(library, metadata.as_ref(), ctx.client.clone(), &storage_dir);
|
||||
let rebuild = rebuild_after_scan(
|
||||
processor.scan_for_target_rebuild(),
|
||||
|result| {
|
||||
if result.errors > 0 {
|
||||
log::error!("Library rescan failed for input {input_id}: incomplete scan; {result:?}");
|
||||
}
|
||||
ctx.events.emit(EventMessage::PlaylistUpdateProgress(
|
||||
PlaylistUpdateProgressEvent::for_run_input(
|
||||
ctx.run_id.clone(),
|
||||
ctx.execution_order,
|
||||
input_id,
|
||||
"Library rescan",
|
||||
"Library scan result",
|
||||
)
|
||||
.with_library_scan_result(result)
|
||||
.with_detail(PlaylistUpdateProgressDetail::LibraryScanResult),
|
||||
));
|
||||
},
|
||||
|| {
|
||||
progress(
|
||||
"Library rescan completed; loading input data and rebuilding selected targets",
|
||||
PlaylistUpdateProgressDetail::LibraryRescanCompleted,
|
||||
);
|
||||
process_sources(ctx)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
// Retain the existing library permit until the catalog-dependent rebuild finishes.
|
||||
Ok::<_, io::Error>(rebuild.await)
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
log::error!("Library rescan failed for input {input_id}: {error}");
|
||||
if let Some(input) = ctx.config.get_input_by_id(input_id) {
|
||||
super::input_status::persist_input_completion(ctx, input.id, &input.name, PlaylistUpdateState::Failure)
|
||||
.await;
|
||||
}
|
||||
ctx.events.emit(EventMessage::PlaylistUpdateProgress(
|
||||
PlaylistUpdateProgressEvent::input_completed(
|
||||
ctx.run_id.clone(),
|
||||
ctx.execution_order,
|
||||
input_id,
|
||||
PlaylistUpdateState::Failure,
|
||||
"Library rescan",
|
||||
"Library rescan failed; selected targets were not rebuilt",
|
||||
)
|
||||
.with_detail(PlaylistUpdateProgressDetail::LibraryRescanFailed),
|
||||
));
|
||||
(Vec::new(), vec![TuliproxError::RepositoryLibrary(format!("Library rescan failed: {error}"))])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::cell::Cell;
|
||||
|
||||
fn scan_result(errors: usize) -> LibraryScanResult {
|
||||
LibraryScanResult {
|
||||
files_scanned: 0,
|
||||
groups_scanned: 0,
|
||||
files_added: 0,
|
||||
files_updated: 0,
|
||||
files_removed: 0,
|
||||
errors,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_rebuild_waits_for_successful_scan() {
|
||||
let completed = Cell::new(false);
|
||||
let scan = async {
|
||||
tokio::task::yield_now().await;
|
||||
completed.set(true);
|
||||
Ok(scan_result(0))
|
||||
};
|
||||
let rebuilt = rebuild_after_scan(
|
||||
scan,
|
||||
|_| {},
|
||||
|| {
|
||||
assert!(completed.get());
|
||||
"common target pipeline"
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(rebuilt.unwrap(), "common target pipeline");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_errors_and_partial_scan_never_start_target_rebuild() {
|
||||
for result in [Err(io::Error::other("scan failed")), Ok(LibraryScanResult { errors: 1, ..scan_result(0) })] {
|
||||
let rebuilt = Cell::new(false);
|
||||
let reported = std::cell::RefCell::new(None);
|
||||
let expected = result.as_ref().ok().cloned();
|
||||
assert!(rebuild_after_scan(
|
||||
async { result },
|
||||
|scan| *reported.borrow_mut() = Some(scan),
|
||||
|| rebuilt.set(true)
|
||||
)
|
||||
.await
|
||||
.is_err());
|
||||
assert_eq!(*reported.borrow(), expected);
|
||||
assert!(!rebuilt.get());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_update_library_bulk_reuses_catalog_without_scan_or_policy_override() {
|
||||
assert_eq!(LibraryUpdateMode::default(), LibraryUpdateMode::ExistingCatalog);
|
||||
assert!(!LibraryUpdateMode::ExistingCatalog.reloads_input(2));
|
||||
assert!(LibraryUpdateMode::Rescan { input_id: 2 }.reloads_input(2));
|
||||
assert!(!LibraryUpdateMode::Rescan { input_id: 2 }.reloads_input(3));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,725 @@
|
||||
mod manual_update_integration {
|
||||
use super::*;
|
||||
use shared::model::{ConfigRenameDto, ItemField, LibraryConfigDto, LibraryContentType, LibraryScanResult};
|
||||
use tuliprox_core::model::{LibraryConfig, LibraryScanDirectory, M3uTargetOutput, MetadataUpdateConfig};
|
||||
|
||||
async fn library_context(root: &Path, order: ProcessingOrder) -> PlaylistProcessingContext<CollectSink> {
|
||||
let ctx = processing_context_with_events(root, CollectSink::default());
|
||||
let media = root.join("media");
|
||||
tokio::fs::create_dir_all(&media).await.unwrap();
|
||||
tokio::fs::write(media.join("Library Movie (2025).mp4"), b"fixture").await.unwrap();
|
||||
let companion_path = root.join("companion.m3u");
|
||||
tokio::fs::write(&companion_path, "#EXTM3U\n#EXTINF:-1,Companion\nhttp://stream.example/companion\n")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
let mut library = LibraryConfig::from(&LibraryConfigDto { enabled: true, ..LibraryConfigDto::default() });
|
||||
library.metadata.fallback_to_filename = true;
|
||||
library.thumbnails.enabled = false;
|
||||
library.scan_directories = vec![LibraryScanDirectory {
|
||||
enabled: true,
|
||||
path: media.to_string_lossy().into_owned(),
|
||||
content_type: LibraryContentType::Movie,
|
||||
recursive: true,
|
||||
}];
|
||||
config.library = Some(library);
|
||||
let mut metadata = MetadataUpdateConfig::default();
|
||||
metadata.ffprobe.enabled = false;
|
||||
metadata.tmdb.enabled = false;
|
||||
config.metadata_update = Some(metadata);
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
let library_input = Arc::new(ConfigInput {
|
||||
id: 2,
|
||||
name: "local".intern(),
|
||||
input_type: InputType::Library,
|
||||
enabled: true,
|
||||
cache_duration_seconds: 3600,
|
||||
..ConfigInput::default()
|
||||
});
|
||||
let companion = Arc::new(ConfigInput {
|
||||
id: 3,
|
||||
name: "companion".intern(),
|
||||
input_type: InputType::M3u,
|
||||
enabled: true,
|
||||
url: companion_path.to_string_lossy().into_owned(),
|
||||
..ConfigInput::default()
|
||||
});
|
||||
let mut target = ConfigTarget::from(&ConfigTargetDto {
|
||||
id: 20,
|
||||
enabled: true,
|
||||
name: "selected".into(),
|
||||
processing_order: order,
|
||||
rename: Some(vec![ConfigRenameDto {
|
||||
field: ItemField::Name,
|
||||
pattern: "^MAPPED$".into(),
|
||||
new_name: "RENAMED".into(),
|
||||
t_pattern: None,
|
||||
}]),
|
||||
..ConfigTargetDto::default()
|
||||
});
|
||||
target.output = vec![TargetOutput::M3u(M3uTargetOutput {
|
||||
filename: Some(root.join("selected.m3u").to_string_lossy().into_owned()),
|
||||
include_type_in_url: false,
|
||||
mask_redirect_url: false,
|
||||
filter: None,
|
||||
})];
|
||||
target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into();
|
||||
let mapped = super::super::mapping_stage::build_target(
|
||||
vec![super::super::mapping_stage::build_mapping("manual", MappingStage::Processing, r#"@Name = "MAPPED""#)],
|
||||
false,
|
||||
);
|
||||
target.mapping = mapped.mapping;
|
||||
let mut unselected = target.clone();
|
||||
unselected.id = 21;
|
||||
unselected.name = "unselected".into();
|
||||
unselected.output = vec![TargetOutput::M3u(M3uTargetOutput {
|
||||
filename: Some(root.join("unselected.m3u").to_string_lossy().into_owned()),
|
||||
include_type_in_url: false,
|
||||
mask_redirect_url: false,
|
||||
filter: None,
|
||||
})];
|
||||
ctx.config.sources.store(Arc::new(SourcesConfig {
|
||||
inputs: vec![library_input.clone(), companion.clone()],
|
||||
sources: vec![ConfigSource {
|
||||
inputs: vec![library_input.name.clone(), companion.name.clone()],
|
||||
targets: vec![Arc::new(target), Arc::new(unselected)],
|
||||
}],
|
||||
..SourcesConfig::default()
|
||||
}));
|
||||
ctx
|
||||
}
|
||||
|
||||
fn rescan_run(ctx: &PlaylistProcessingContext<CollectSink>, guard: UpdateGuard) -> ProcessingRun<CollectSink> {
|
||||
ProcessingRun::for_run(
|
||||
"manual-library-run".into(),
|
||||
ctx.client.clone(),
|
||||
ctx.config.clone(),
|
||||
Arc::new(ProcessTargets {
|
||||
enabled: true,
|
||||
inputs: vec![2, 3],
|
||||
targets: vec![20],
|
||||
target_names: vec!["selected".into()],
|
||||
}),
|
||||
ctx.events.clone(),
|
||||
)
|
||||
.with_update_guard(guard)
|
||||
.with_manual_input_update(Some(InputUpdateRequest { input_id: 2, action: InputUpdateAction::Rescan }))
|
||||
}
|
||||
|
||||
fn scan_results(ctx: &PlaylistProcessingContext<CollectSink>) -> Vec<LibraryScanResult> {
|
||||
ctx.events
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) => {
|
||||
if progress.library_scan_result.is_some() {
|
||||
assert_eq!(
|
||||
progress.run_id.as_ref().map(PlaylistUpdateRunId::as_ref),
|
||||
Some("manual-library-run")
|
||||
);
|
||||
assert!(progress.execution_order.is_some());
|
||||
}
|
||||
progress.library_scan_result.clone()
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn assert_run_state(ctx: &PlaylistProcessingContext<CollectSink>, state: PlaylistUpdateState) {
|
||||
let events = ctx.events.0.lock().unwrap();
|
||||
assert!(
|
||||
events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == state)),
|
||||
"{events:?}"
|
||||
);
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) && progress.state == Some(state))), "{events:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_empty_rescan_persists_zero_to_zero_and_nonzero_to_zero_with_companion() {
|
||||
for disk in [false, true] {
|
||||
for previous_population in [false, true] {
|
||||
for companion in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
config.disk_based_processing = disk;
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
if !companion {
|
||||
let mut sources = (**ctx.config.sources.load()).clone();
|
||||
sources.sources[0].inputs.retain(|name| name.as_ref() == "local");
|
||||
ctx.config.sources.store(Arc::new(sources));
|
||||
}
|
||||
if previous_population {
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
ctx.events.0.lock().unwrap().clear();
|
||||
}
|
||||
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
for name in std::iter::once("local").chain(companion.then_some("companion")) {
|
||||
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), name).await;
|
||||
let status = input_cache::load_input_status(&path);
|
||||
assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Success, "{name}");
|
||||
}
|
||||
assert_eq!(
|
||||
scan_results(&ctx),
|
||||
vec![LibraryScanResult {
|
||||
files_scanned: 0,
|
||||
groups_scanned: 0,
|
||||
files_added: 0,
|
||||
files_updated: 0,
|
||||
files_removed: usize::from(previous_population),
|
||||
errors: 0,
|
||||
}]
|
||||
);
|
||||
let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
|
||||
assert_eq!(output.matches("#EXTINF").count(), usize::from(companion), "{output}");
|
||||
assert_eq!(output.contains("http://stream.example/companion"), companion);
|
||||
assert!(!output.contains("Library%20Movie") && !output.contains("file://"));
|
||||
assert!(!temp.path().join("unselected.m3u").exists());
|
||||
let input = ctx.config.sources.load().inputs[0].clone();
|
||||
let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert!(persisted.is_empty(), "Empty Library must replace its old persisted input");
|
||||
assert!(!ctx.events.0.lock().unwrap().iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.message.contains("playlist is empty"))));
|
||||
// Same canonical empty input remains readable on subsequent cache/reuse paths.
|
||||
let first = download_input(&ctx, &input, false).await;
|
||||
assert!(first.errors.is_empty() && first.storage_error.is_none());
|
||||
let mut reuse = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(reuse.job_state(), InputJobState::Ready);
|
||||
assert_eq!(reuse.input_telemetry, None, "In-run reuse must not emit another acquisition");
|
||||
assert_eq!(scan_results(&ctx).len(), 1, "Reuse must not manufacture a second rescan");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coderabbit_library_unavailable_is_not_an_authoritative_empty_catalog() {
|
||||
for disabled in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
let mut input = (*ctx.config.sources.load().inputs[0]).clone();
|
||||
input.cache_duration_seconds = 0;
|
||||
let input = Arc::new(input);
|
||||
let storage = input_storage_path(&ctx, &input).await;
|
||||
let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name);
|
||||
let before = tokio::fs::read(&file).await.unwrap();
|
||||
let output = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap();
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
if disabled {
|
||||
config.library.as_mut().unwrap().enabled = false;
|
||||
} else {
|
||||
config.library = None;
|
||||
}
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
|
||||
let mut result = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(result.job_state(), InputJobState::Failed);
|
||||
assert_eq!(result.update_state(), PlaylistUpdateState::Failure);
|
||||
assert!(!result.authoritative_library);
|
||||
assert!(!result.errors.is_empty());
|
||||
assert_eq!(tokio::fs::read(&file).await.unwrap(), before);
|
||||
assert_eq!(tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(), output);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coderabbit_library_missing_input_triggers_catalog_recovery_not_empty_success() {
|
||||
for disk in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
config.disk_based_processing = disk;
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
let input = ctx.config.sources.load().inputs[0].clone();
|
||||
let storage = input_storage_path(&ctx, &input).await;
|
||||
let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name);
|
||||
tokio::fs::remove_file(&file).await.unwrap();
|
||||
|
||||
assert!(load_input_playlist(&ctx.config, &input, None).await.is_err());
|
||||
let mut recovered = download_input(&ctx, &input, false).await;
|
||||
assert_eq!(recovered.job_state(), InputJobState::Ready);
|
||||
assert!(recovered.errors.is_empty() && recovered.storage_error.is_none());
|
||||
assert!(!recovered.source.is_empty(), "missing cache must be repaired from the populated catalog");
|
||||
assert!(file.exists());
|
||||
let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap();
|
||||
assert!(!persisted.is_empty());
|
||||
|
||||
let media_server = ConfigInput { input_type: InputType::Plex, ..(*input).clone() };
|
||||
assert!(
|
||||
load_input_playlist(&ctx.config, &media_server, None).await.is_err(),
|
||||
"the shared loader must not turn a missing Plex cache into a successful empty input"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_scan_errors_keep_real_result_and_block_rebuild() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
config.library.as_mut().unwrap().metadata.fallback_to_filename = false;
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Failure);
|
||||
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await;
|
||||
assert_eq!(
|
||||
input_cache::load_input_status(&path).last_input_update.unwrap().state,
|
||||
PlaylistUpdateState::Failure
|
||||
);
|
||||
assert_eq!(
|
||||
scan_results(&ctx),
|
||||
vec![LibraryScanResult {
|
||||
files_scanned: 1,
|
||||
groups_scanned: 1,
|
||||
files_added: 0,
|
||||
files_updated: 0,
|
||||
files_removed: 0,
|
||||
errors: 1,
|
||||
}]
|
||||
);
|
||||
assert!(!temp.path().join("selected.m3u").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_persisted_input_status_keeps_failed_companion_separate() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "companion").await;
|
||||
assert_eq!(
|
||||
input_cache::load_input_status(&path).last_input_update.unwrap().state,
|
||||
PlaylistUpdateState::Success
|
||||
);
|
||||
let before = std::fs::read(temp.path().join("selected.m3u")).unwrap();
|
||||
tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_eq!(
|
||||
input_cache::load_input_status(&path).last_input_update.unwrap().state,
|
||||
PlaylistUpdateState::Failure
|
||||
);
|
||||
let library_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await;
|
||||
assert_eq!(
|
||||
input_cache::load_input_status(&library_path).last_input_update.unwrap().state,
|
||||
PlaylistUpdateState::Success
|
||||
);
|
||||
assert_eq!(std::fs::read(temp.path().join("selected.m3u")).unwrap(), before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_empty_xtream_and_strm_use_existing_publish_and_keep_companions() {
|
||||
use tuliprox_repository::{xtream_get_file_path, xtream_get_storage_path, BPlusTreeQuery};
|
||||
for companion in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
tokio::fs::write(
|
||||
temp.path().join("companion.m3u"),
|
||||
"#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sources = (**ctx.config.sources.load()).clone();
|
||||
if !companion {
|
||||
sources.sources[0].inputs.retain(|name| name.as_ref() == "local");
|
||||
}
|
||||
let target = Arc::make_mut(&mut sources.sources[0].targets[0]);
|
||||
target.output.extend([
|
||||
TargetOutput::Xtream(XtreamTargetOutput {
|
||||
flags: XtreamTargetFlagsSet::new(),
|
||||
trakt: None,
|
||||
filter: None,
|
||||
}),
|
||||
TargetOutput::Strm(tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto {
|
||||
directory: temp.path().join("strm").to_string_lossy().into_owned(),
|
||||
flat: true,
|
||||
..shared::model::StrmTargetOutputDto::default()
|
||||
})),
|
||||
]);
|
||||
ctx.config.sources.store(Arc::new(sources));
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
ctx.events.0.lock().unwrap().clear();
|
||||
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
let config = ctx.config.config.load();
|
||||
let xtream = xtream_get_storage_path(&config, "selected").unwrap();
|
||||
for (cluster, expected) in
|
||||
[(XtreamCluster::Live, 0), (XtreamCluster::Video, usize::from(companion)), (XtreamCluster::Series, 0)]
|
||||
{
|
||||
let path = xtream_get_file_path(&xtream, cluster);
|
||||
if !path.exists() {
|
||||
assert_eq!(expected, 0);
|
||||
continue;
|
||||
}
|
||||
let mut query = BPlusTreeQuery::<u32, shared::model::XtreamPlaylistItem>::try_new(&path).unwrap();
|
||||
assert_eq!(query.iter().count(), expected, "{cluster:?}");
|
||||
}
|
||||
let entries = strm_artifacts(&temp.path().join("strm")).await;
|
||||
assert_eq!(entries.len(), usize::from(companion), "{entries:?}");
|
||||
if companion {
|
||||
assert!(entries[0].1.contains("http://stream.example/companion"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_empty_permission_does_not_loosen_other_provider_inputs() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
for input_type in [InputType::M3u, InputType::Xtream, InputType::Stalker, InputType::Plex] {
|
||||
let input =
|
||||
ConfigInput { input_type, name: format!("empty-{input_type:?}").intern(), ..ConfigInput::default() };
|
||||
let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, Vec::new()).await;
|
||||
assert!(error.is_some(), "{input_type:?} must retain its generic empty-input guard");
|
||||
}
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap();
|
||||
ctx.events.0.lock().unwrap().clear();
|
||||
tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
{
|
||||
let events = ctx.events.0.lock().unwrap();
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure)));
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure))));
|
||||
}
|
||||
assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap());
|
||||
// An authoritatively empty Library must not mask a failed companion either.
|
||||
ctx.events.0.lock().unwrap().clear();
|
||||
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
{
|
||||
let events = ctx.events.0.lock().unwrap();
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure)));
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure))));
|
||||
}
|
||||
assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap());
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum LibraryFilterBoundary {
|
||||
Processing,
|
||||
Persist,
|
||||
M3u,
|
||||
Strm,
|
||||
}
|
||||
|
||||
async fn strm_artifacts(root: &Path) -> Vec<(PathBuf, String)> {
|
||||
let mut directories = vec![root.to_path_buf()];
|
||||
let mut artifacts = Vec::new();
|
||||
while let Some(directory) = directories.pop() {
|
||||
let mut entries = tokio::fs::read_dir(directory).await.unwrap();
|
||||
while let Some(entry) = entries.next_entry().await.unwrap() {
|
||||
if entry.file_type().await.unwrap().is_dir() {
|
||||
directories.push(entry.path());
|
||||
} else if entry.path().extension().is_some_and(|extension| extension == "strm") {
|
||||
artifacts.push((entry.path(), tokio::fs::read_to_string(entry.path()).await.unwrap()));
|
||||
}
|
||||
}
|
||||
}
|
||||
artifacts.sort();
|
||||
artifacts
|
||||
}
|
||||
|
||||
async fn assert_library_filtered_empty_publication(boundary: LibraryFilterBoundary) {
|
||||
for disk in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
config.disk_based_processing = disk;
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
tokio::fs::write(
|
||||
temp.path().join("companion.m3u"),
|
||||
"#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sources = (**ctx.config.sources.load()).clone();
|
||||
let target = Arc::make_mut(&mut sources.sources[0].targets[0]);
|
||||
target.rename = None;
|
||||
target.mapping = Arc::default();
|
||||
let library_only = get_filter(r#"Name ~ "^Library Movie.*""#, None).unwrap();
|
||||
if matches!(boundary, LibraryFilterBoundary::Processing) {
|
||||
target.filter = library_only.clone().into();
|
||||
}
|
||||
if matches!(boundary, LibraryFilterBoundary::Persist) {
|
||||
target.filter.persist = Some(library_only.clone());
|
||||
}
|
||||
if let TargetOutput::M3u(output) = &mut target.output[0] {
|
||||
output.filter = matches!(boundary, LibraryFilterBoundary::M3u).then(|| library_only.clone());
|
||||
}
|
||||
let strm_root = temp.path().join("strm");
|
||||
let mut strm = tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto {
|
||||
directory: strm_root.to_string_lossy().into_owned(),
|
||||
flat: true,
|
||||
cleanup: true,
|
||||
..shared::model::StrmTargetOutputDto::default()
|
||||
});
|
||||
strm.filter = matches!(boundary, LibraryFilterBoundary::Strm).then_some(library_only);
|
||||
target.output.push(TargetOutput::Strm(strm));
|
||||
ctx.config.sources.store(Arc::new(sources));
|
||||
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
let m3u_path = temp.path().join("selected.m3u");
|
||||
let before_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap();
|
||||
let before_strm = strm_artifacts(&strm_root).await;
|
||||
let m3u_keeps_companion = matches!(boundary, LibraryFilterBoundary::Strm);
|
||||
let strm_keeps_companion = matches!(boundary, LibraryFilterBoundary::M3u);
|
||||
assert_eq!(before_m3u.matches("#EXTINF").count(), 1 + usize::from(m3u_keeps_companion));
|
||||
assert_eq!(before_strm.len(), 1 + usize::from(strm_keeps_companion));
|
||||
assert_eq!(before_m3u.contains("http://stream.example/companion"), m3u_keeps_companion);
|
||||
assert_eq!(
|
||||
before_strm.iter().any(|(_, body)| body.contains("http://stream.example/companion")),
|
||||
strm_keeps_companion
|
||||
);
|
||||
|
||||
ctx.events.0.lock().unwrap().clear();
|
||||
tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
let after_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap();
|
||||
let after_strm = strm_artifacts(&strm_root).await;
|
||||
println!("Library filter {boundary:?}, disk={disk}: M3U before={before_m3u:?}, after={after_m3u:?}; STRM before={before_strm:?}, after={after_strm:?}");
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Success);
|
||||
assert_eq!(
|
||||
scan_results(&ctx),
|
||||
vec![LibraryScanResult {
|
||||
files_scanned: 0,
|
||||
groups_scanned: 0,
|
||||
files_added: 0,
|
||||
files_updated: 0,
|
||||
files_removed: 1,
|
||||
errors: 0
|
||||
}]
|
||||
);
|
||||
assert_ne!(before_m3u, after_m3u, "old Library entries must really be replaced");
|
||||
assert_eq!(after_m3u.matches("#EXTINF").count(), usize::from(m3u_keeps_companion));
|
||||
assert_eq!(after_m3u.contains("http://stream.example/companion"), m3u_keeps_companion);
|
||||
assert_eq!(after_strm.len(), usize::from(strm_keeps_companion));
|
||||
for (path, contents) in &before_strm {
|
||||
if contents.contains("http://stream.example/companion") {
|
||||
assert!(
|
||||
after_strm.contains(&(path.clone(), contents.clone())),
|
||||
"valid companion must be retained byte-for-byte"
|
||||
);
|
||||
} else {
|
||||
assert!(!path.exists(), "cleanup must remove old Library artifact {path:?}");
|
||||
}
|
||||
}
|
||||
// The existing M3U database is refreshed too, not only the client text file.
|
||||
let target_path =
|
||||
tuliprox_repository::get_target_storage_path(&ctx.config.config.load(), "selected").unwrap();
|
||||
let mut db = tuliprox_repository::BPlusTreeQuery::<u32, shared::model::M3uPlaylistItem>::try_new(
|
||||
&tuliprox_repository::m3u_get_file_path_for_db(&target_path),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(db.iter().count(), usize::from(m3u_keeps_companion));
|
||||
assert!(!temp.path().join("unselected.m3u").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_processing_filter_replaces_old_output_after_last_movie_removed() {
|
||||
assert_library_filtered_empty_publication(LibraryFilterBoundary::Processing).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_persist_filter_replaces_old_output_after_last_movie_removed() {
|
||||
assert_library_filtered_empty_publication(LibraryFilterBoundary::Persist).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_m3u_output_filter_replaces_old_output_and_preserves_unfiltered_strm() {
|
||||
assert_library_filtered_empty_publication(LibraryFilterBoundary::M3u).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_strm_output_filter_cleans_old_files_and_preserves_unfiltered_m3u() {
|
||||
assert_library_filtered_empty_publication(LibraryFilterBoundary::Strm).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_empty_does_not_authorize_foreign_forced_empty_target() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
let sources = ctx.config.sources.load_full();
|
||||
let mut playlists = sources
|
||||
.inputs
|
||||
.iter()
|
||||
.map(|input| FetchedPlaylist { input, source: MemoryPlaylistSource::default().into_source(), epg: None })
|
||||
.collect::<Vec<_>>();
|
||||
let prepared = prepare_playlist_for_target(
|
||||
&ctx,
|
||||
&mut playlists,
|
||||
&sources.sources[0].targets[0],
|
||||
&mut HashMap::new(),
|
||||
&mut Vec::new(),
|
||||
ClusterFlags::Vod,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(prepared.library_empty, tuliprox_repository::LibraryEmptyPublication::Contribution);
|
||||
let (result, _) = finalize_prepared_target(Arc::new(ctx), prepared).await;
|
||||
assert!(result.is_err(), "Library must not relax the existing non-Xtream force-empty output guard");
|
||||
assert!(!temp.path().join("selected.m3u").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_incomplete_discovery_has_no_invented_scan_result_or_rebuild() {
|
||||
for failure in ["missing", "not-directory", "partial", "no-directories", "corrupt-catalog"] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap();
|
||||
let status_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await;
|
||||
let cache_before = input_cache::load_input_status(&status_path).clusters;
|
||||
ctx.events.0.lock().unwrap().clear();
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
let library = config.library.as_mut().unwrap();
|
||||
match failure {
|
||||
"missing" => {
|
||||
library.scan_directories[0].path = temp.path().join("missing").to_string_lossy().into_owned();
|
||||
}
|
||||
"not-directory" => {
|
||||
library.scan_directories[0].path =
|
||||
temp.path().join("media/Library Movie (2025).mp4").to_string_lossy().into_owned();
|
||||
}
|
||||
"partial" => {
|
||||
let mut missing = library.scan_directories[0].clone();
|
||||
missing.path = temp.path().join("missing").to_string_lossy().into_owned();
|
||||
library.scan_directories.push(missing);
|
||||
}
|
||||
"no-directories" => library.scan_directories.clear(),
|
||||
"corrupt-catalog" => {
|
||||
let path = tuliprox_library::library::resolve_metadata_storage_path(
|
||||
config.metadata_update.as_ref(),
|
||||
&config.storage_dir,
|
||||
)
|
||||
.join("library/broken.json");
|
||||
tokio::fs::write(path, "invalid json").await.unwrap();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
assert_run_state(&ctx, PlaylistUpdateState::Failure);
|
||||
assert!(scan_results(&ctx).is_empty(), "{failure}");
|
||||
let status = input_cache::load_input_status(&status_path);
|
||||
assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Failure, "{failure}");
|
||||
assert_eq!(status.clusters, cache_before, "failed rescan must not mutate cache validity");
|
||||
assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_selected_target_keeps_companion_and_real_processing_order() {
|
||||
for (order, expected) in [(ProcessingOrder::Frm, "MAPPED"), (ProcessingOrder::Fmr, "RENAMED")] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), order).await;
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
|
||||
assert_eq!(output.matches("#EXTINF").count(), 2, "{output}");
|
||||
assert!(output.contains("http://stream.example/companion"), "{output}");
|
||||
assert!(output.contains(expected), "{output}");
|
||||
assert!(!temp.path().join("unselected.m3u").exists());
|
||||
let events = ctx.events.0.lock().unwrap();
|
||||
let scan = events
|
||||
.iter()
|
||||
.position(|event| {
|
||||
matches!(event, EventMessage::PlaylistUpdateProgress(progress)
|
||||
if progress.input_id == Some(2) && progress.message.starts_with("Library rescan completed"))
|
||||
})
|
||||
.unwrap();
|
||||
let acquired = events
|
||||
.iter()
|
||||
.position(|event| {
|
||||
matches!(event, EventMessage::PlaylistUpdateProgress(progress)
|
||||
if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success))
|
||||
})
|
||||
.unwrap();
|
||||
let published = events
|
||||
.iter()
|
||||
.position(|event| {
|
||||
matches!(event, EventMessage::PlaylistUpdate(summary)
|
||||
if summary.state == PlaylistUpdateState::Success)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(scan < acquired && acquired < published);
|
||||
assert!(events.iter().all(|event| !matches!(event, EventMessage::PlaylistUpdateProgress(progress)
|
||||
if progress.input_id == Some(2) && progress.input_telemetry.as_ref().is_some_and(|telemetry|
|
||||
!telemetry.clusters.is_empty() || telemetry.refresh_policy != InputRefreshPolicy::NORMAL))));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_unreadable_catalog_and_busy_scan_do_not_publish() {
|
||||
for busy in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
let guard = UpdateGuard::new();
|
||||
let _busy = busy.then(|| guard.try_library().unwrap());
|
||||
if !busy {
|
||||
let mut config = (**ctx.config.config.load()).clone();
|
||||
config.library.as_mut().unwrap().scan_directories[0].path =
|
||||
temp.path().join("missing").to_string_lossy().into_owned();
|
||||
ctx.config.config.store(Arc::new(config));
|
||||
}
|
||||
exec_processing(rescan_run(&ctx, guard.clone())).await;
|
||||
assert!(!temp.path().join("selected.m3u").exists());
|
||||
assert!(!temp.path().join("unselected.m3u").exists());
|
||||
assert!(ctx
|
||||
.events
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|event| matches!(event, EventMessage::PlaylistUpdate(summary)
|
||||
if summary.state == PlaylistUpdateState::Failure)));
|
||||
assert!(guard.try_playlist().is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_input_success_remains_separate_from_target_publish_failure() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
tokio::fs::create_dir(temp.path().join("selected.m3u")).await.unwrap();
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
let events = ctx.events.0.lock().unwrap();
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress)
|
||||
if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success))));
|
||||
assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary)
|
||||
if summary.state == PlaylistUpdateState::Failure)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_update_library_rescan_reloads_valid_cache_but_bulk_keeps_catalog() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let ctx = library_context(temp.path(), ProcessingOrder::Frm).await;
|
||||
exec_processing(rescan_run(&ctx, UpdateGuard::new())).await;
|
||||
tokio::fs::write(temp.path().join("media/Second Movie (2025).mp4"), b"fixture").await.unwrap();
|
||||
// Ordinary bulk does not discover the new file and does not acquire a library permit.
|
||||
let guard = UpdateGuard::new();
|
||||
let standalone_scan = guard.try_library().unwrap();
|
||||
exec_processing(
|
||||
ProcessingRun::new(ctx.client.clone(), ctx.config.clone(), ctx.user_targets.clone(), ctx.events.clone())
|
||||
.with_update_guard(guard.clone()),
|
||||
)
|
||||
.await;
|
||||
let bulk = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
|
||||
assert_eq!(bulk.matches("#EXTINF").count(), 2);
|
||||
drop(standalone_scan);
|
||||
exec_processing(rescan_run(&ctx, guard)).await;
|
||||
let rescanned = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap();
|
||||
assert_eq!(rescanned.matches("#EXTINF").count(), 3, "{rescanned}");
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ use super::providers::{LibraryProvider, PlexProvider, StalkerProvider, XmltvEpgP
|
||||
use crate::{
|
||||
fetched_playlist::FetchedPlaylist,
|
||||
input_cache,
|
||||
input_cache::ClusterState,
|
||||
metadata_sink::{MetadataUpdateSink, NoopMetadataSink},
|
||||
parser::xmltv::{flatten_tvguide, merge_epg_trees, EpgMergeAccumulator, TVGuide},
|
||||
playlist_watch::{process_group_watch, process_target_groups_watch},
|
||||
@@ -24,10 +23,14 @@ use shared::{
|
||||
error::{get_errors_notify_message, TuliproxError},
|
||||
foundation::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider},
|
||||
model::{
|
||||
ClusterFlags, ConfigTargetOptions, CounterModifier, EventMessage, EventSink, FieldGet, FieldSet, InputStats,
|
||||
InputType, MappingStage, PipelineStats, PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistStats,
|
||||
PlaylistUpdateProgressEvent, PlaylistUpdateSummary, ProviderFetchFailure, SourceStats, StreamProperties,
|
||||
TargetStats, UUIDType, WatchDisabled, WatchDisabledReason, WatchUnmatched, XtreamCluster,
|
||||
ClusterFlags, ConfigTargetOptions, CounterModifier, EventMessage, EventSink, FieldGet, FieldSet,
|
||||
InputRefreshOverride, InputRefreshPolicy, InputStats, InputType, InputUpdateAction, InputUpdateRequest,
|
||||
MappingStage, PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateQualityDecision,
|
||||
PersistedPlaylistUpdateQualitySnapshot, PersistedPlaylistUpdateTechnicalState, PipelineStats, PlaylistGroup,
|
||||
PlaylistItem, PlaylistItemType, PlaylistStats, PlaylistUpdateClusterDecision, PlaylistUpdateClusterTelemetry,
|
||||
PlaylistUpdateDataSource, PlaylistUpdateInputTelemetry, PlaylistUpdateProgressEvent, PlaylistUpdateRunId,
|
||||
PlaylistUpdateRunOrder, PlaylistUpdateState, PlaylistUpdateSummary, ProviderFetchFailure, SourceStats,
|
||||
StreamProperties, TargetStats, UUIDType, WatchDisabled, WatchDisabledReason, WatchUnmatched, XtreamCluster,
|
||||
},
|
||||
utils::{create_alias_uuid, interner_gc, sanitize_sensitive_info, Internable},
|
||||
};
|
||||
@@ -35,7 +38,7 @@ use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
future::Future,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Weak},
|
||||
sync::{atomic::AtomicU64, Arc, Weak},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::{
|
||||
@@ -44,10 +47,10 @@ use tokio::{
|
||||
};
|
||||
use tuliprox_core::{
|
||||
model::{
|
||||
is_valid, retain_filtered_playlist, AppConfig, CompiledMapping, ConfigFavourites, ConfigInput,
|
||||
ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, Epg, FilterOutcome, MappingProgram,
|
||||
ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig, TransformStage, UpdateGuard,
|
||||
UpdateTask,
|
||||
is_valid, retain_filtered_playlist, AppConfig, ClusterForceUpdate, ClusterUpdateRejection, CompiledMapping,
|
||||
ConfigFavourites, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, Epg,
|
||||
FilterOutcome, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig,
|
||||
TransformStage, UpdateGuard, UpdateTask,
|
||||
},
|
||||
utils::{debug_if_enabled, log_memory_snapshot, trace_if_enabled, StepMeasure, StepMeasureCallback},
|
||||
};
|
||||
@@ -62,15 +65,28 @@ use tuliprox_iptv::{
|
||||
xtream,
|
||||
};
|
||||
use tuliprox_repository::{
|
||||
load_input_playlist, persist_input_playlist, persist_playlist, CategoryKey, MemoryPlaylistSource, PlaylistSource,
|
||||
PlaylistStorageState,
|
||||
load_input_playlist, persist_input_playlist_with_options, persist_playlist, CategoryKey,
|
||||
InputPlaylistPersistOptions, MemoryPlaylistSource, PlaylistSource, PlaylistStorageState,
|
||||
TargetPlaylistPersistOptions,
|
||||
};
|
||||
use tuliprox_session::ActiveProviderManager;
|
||||
|
||||
const PLAYLIST_UPDATE_MAX_DURATION_SECS: u64 = 3600;
|
||||
const MAX_CONCURRENT_TARGET_FINALIZERS: usize = 2;
|
||||
static NEXT_PLAYLIST_UPDATE_RUN_ORDER: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
/// Allocates the order at which a run entered the serial execution boundary.
|
||||
#[must_use]
|
||||
pub fn next_playlist_update_run_order() -> PlaylistUpdateRunOrder {
|
||||
PlaylistUpdateRunOrder::from(NEXT_PLAYLIST_UPDATE_RUN_ORDER.fetch_add(1, std::sync::atomic::Ordering::SeqCst))
|
||||
}
|
||||
|
||||
mod fetch_outcome;
|
||||
mod ingest;
|
||||
mod input_status;
|
||||
mod m3u_quality;
|
||||
mod manual_update;
|
||||
use manual_update::LibraryUpdateMode;
|
||||
mod target;
|
||||
mod transform;
|
||||
|
||||
@@ -123,6 +139,7 @@ pub struct ProcessingRun<
|
||||
M: MetadataUpdateSink = NoopMetadataSink,
|
||||
> {
|
||||
client: reqwest::Client,
|
||||
run_id: PlaylistUpdateRunId,
|
||||
app_config: Arc<AppConfig>,
|
||||
targets: Arc<ProcessTargets>,
|
||||
events: E,
|
||||
@@ -134,12 +151,27 @@ pub struct ProcessingRun<
|
||||
metadata_manager: Option<Arc<M>>,
|
||||
pre_processed_inputs: Option<HashSet<Arc<str>>>,
|
||||
acquired_permit: Option<tuliprox_core::model::UpdateGuardPermit>,
|
||||
input_refresh: Option<InputRefreshOverride>,
|
||||
library_update_mode: LibraryUpdateMode,
|
||||
}
|
||||
|
||||
impl<E: EventSink + Clone + 'static> ProcessingRun<E, NoBootstrap, NoopMetadataSink> {
|
||||
pub fn new(client: reqwest::Client, app_config: Arc<AppConfig>, targets: Arc<ProcessTargets>, events: E) -> Self {
|
||||
Self::for_run(PlaylistUpdateRunId::generate(), client, app_config, targets, events)
|
||||
}
|
||||
|
||||
/// Creates a run for an identity already allocated by an accepted queue request.
|
||||
#[must_use]
|
||||
pub fn for_run(
|
||||
run_id: PlaylistUpdateRunId,
|
||||
client: reqwest::Client,
|
||||
app_config: Arc<AppConfig>,
|
||||
targets: Arc<ProcessTargets>,
|
||||
events: E,
|
||||
) -> Self {
|
||||
Self {
|
||||
client,
|
||||
run_id,
|
||||
app_config,
|
||||
targets,
|
||||
events,
|
||||
@@ -151,6 +183,8 @@ impl<E: EventSink + Clone + 'static> ProcessingRun<E, NoBootstrap, NoopMetadataS
|
||||
metadata_manager: None,
|
||||
pre_processed_inputs: None,
|
||||
acquired_permit: None,
|
||||
input_refresh: None,
|
||||
library_update_mode: LibraryUpdateMode::ExistingCatalog,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -164,6 +198,7 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
|
||||
pub fn with_bootstrap<B2: UpdateBootstrap>(self, bootstrap: B2) -> ProcessingRun<E, B2, M> {
|
||||
ProcessingRun {
|
||||
client: self.client,
|
||||
run_id: self.run_id,
|
||||
app_config: self.app_config,
|
||||
targets: self.targets,
|
||||
events: self.events,
|
||||
@@ -175,6 +210,8 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
|
||||
metadata_manager: self.metadata_manager,
|
||||
pre_processed_inputs: self.pre_processed_inputs,
|
||||
acquired_permit: self.acquired_permit,
|
||||
input_refresh: self.input_refresh,
|
||||
library_update_mode: self.library_update_mode,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,6 +248,7 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
|
||||
pub fn with_metadata_manager<M2: MetadataUpdateSink>(self, manager: Arc<M2>) -> ProcessingRun<E, B, M2> {
|
||||
ProcessingRun {
|
||||
client: self.client,
|
||||
run_id: self.run_id,
|
||||
app_config: self.app_config,
|
||||
targets: self.targets,
|
||||
events: self.events,
|
||||
@@ -222,6 +260,8 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
|
||||
metadata_manager: Some(manager),
|
||||
pre_processed_inputs: self.pre_processed_inputs,
|
||||
acquired_permit: self.acquired_permit,
|
||||
input_refresh: self.input_refresh,
|
||||
library_update_mode: self.library_update_mode,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,6 +280,30 @@ impl<E: EventSink + Clone + 'static, B: UpdateBootstrap, M: MetadataUpdateSink>
|
||||
self.acquired_permit = permit.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Carries a validated card action through the existing serial run.
|
||||
#[must_use]
|
||||
pub fn with_manual_input_update(mut self, request: Option<InputUpdateRequest>) -> Self {
|
||||
if let Some(request) = request {
|
||||
match request.action {
|
||||
InputUpdateAction::Provider(policy) => {
|
||||
self.input_refresh = Some(InputRefreshOverride { input_id: request.input_id, policy });
|
||||
}
|
||||
InputUpdateAction::Rescan => {
|
||||
self.library_update_mode = LibraryUpdateMode::Rescan { input_id: request.input_id };
|
||||
self.input_refresh = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Applies one manual policy override without changing the input configuration.
|
||||
#[must_use]
|
||||
pub fn with_input_refresh(mut self, input_refresh: impl Into<Option<InputRefreshOverride>>) -> Self {
|
||||
self.input_refresh = input_refresh.into();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
@@ -248,6 +312,7 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
) {
|
||||
let ProcessingRun {
|
||||
client,
|
||||
run_id,
|
||||
app_config,
|
||||
targets,
|
||||
events,
|
||||
@@ -259,6 +324,8 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
metadata_manager,
|
||||
pre_processed_inputs,
|
||||
acquired_permit,
|
||||
input_refresh,
|
||||
library_update_mode,
|
||||
} = run;
|
||||
|
||||
let max_update_duration = Duration::from_secs(PLAYLIST_UPDATE_MAX_DURATION_SECS);
|
||||
@@ -269,7 +336,10 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
Some(permit)
|
||||
} else {
|
||||
warn!("Playlist update lock is closed; update skipped.");
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
|
||||
let execution_order = next_playlist_update_run_order();
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
|
||||
run_id,
|
||||
execution_order,
|
||||
shared::model::PlaylistUpdateState::Failure,
|
||||
)));
|
||||
return;
|
||||
@@ -277,6 +347,7 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let execution_order = next_playlist_update_run_order();
|
||||
|
||||
if playlist_guard.is_some() {
|
||||
if let Some(bootstrap) = bootstrap.as_ref() {
|
||||
@@ -284,7 +355,9 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
error!(
|
||||
"Playlist update bootstrap timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
|
||||
);
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
|
||||
run_id,
|
||||
execution_order,
|
||||
shared::model::PlaylistUpdateState::Failure,
|
||||
)));
|
||||
return;
|
||||
@@ -306,11 +379,14 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
// Initialize Context
|
||||
let ctx = PlaylistProcessingContext {
|
||||
client,
|
||||
run_id: run_id.clone(),
|
||||
execution_order,
|
||||
config: app_config.clone(),
|
||||
user_targets: targets.clone(),
|
||||
events: events.clone(),
|
||||
playlist_state: playlist_state.clone(),
|
||||
processed_inputs: Arc::new(Mutex::new(HashSet::new())),
|
||||
input_completions: Arc::new(Mutex::new(HashMap::new())),
|
||||
input_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||
disabled_headers,
|
||||
provider_manager,
|
||||
@@ -324,17 +400,24 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
StalkerRefreshMode::Complete
|
||||
},
|
||||
partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
input_refresh,
|
||||
library_update_mode,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let process_result =
|
||||
tokio::time::timeout(max_update_duration, std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind())
|
||||
.await;
|
||||
let process_result = tokio::time::timeout(
|
||||
max_update_duration,
|
||||
std::panic::AssertUnwindSafe(manual_update::process_manual_update(&ctx, update_guard.as_ref())).catch_unwind(),
|
||||
)
|
||||
.await;
|
||||
let (stats, errors) = match process_result {
|
||||
Ok(Ok((stats, errors))) => (stats, errors),
|
||||
Ok(Err(_)) => {
|
||||
error!("Playlist processing panicked");
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
|
||||
run_id,
|
||||
execution_order,
|
||||
shared::model::PlaylistUpdateState::Failure,
|
||||
)));
|
||||
return;
|
||||
@@ -343,7 +426,9 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
error!(
|
||||
"Playlist processing timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
|
||||
);
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::state_only(
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run(
|
||||
run_id,
|
||||
execution_order,
|
||||
shared::model::PlaylistUpdateState::Failure,
|
||||
)));
|
||||
return;
|
||||
@@ -373,22 +458,29 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
// refresh notified twice. Subscribers now get one event with everything,
|
||||
// and the bridge renders the single message from it.
|
||||
let error = get_errors_notify_message!(errors, 255);
|
||||
let outcome = if error.is_some() {
|
||||
shared::model::PlaylistUpdateState::Failure
|
||||
} else if ctx.partial_refresh.load(std::sync::atomic::Ordering::Acquire) {
|
||||
shared::model::PlaylistUpdateState::Partial
|
||||
} else {
|
||||
shared::model::PlaylistUpdateState::Success
|
||||
};
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary { state: outcome, stats, error }));
|
||||
let outcome = PlaylistRunSignals {
|
||||
has_error: error.is_some(),
|
||||
has_pending_stalker_refresh: ctx.partial_refresh.load(std::sync::atomic::Ordering::Acquire),
|
||||
has_quality_rejections: ctx.had_quality_rejections.load(std::sync::atomic::Ordering::Acquire),
|
||||
}
|
||||
.state();
|
||||
events.emit(EventMessage::PlaylistUpdate(PlaylistUpdateSummary {
|
||||
run_id: Some(run_id.clone()),
|
||||
execution_order: Some(execution_order),
|
||||
state: outcome,
|
||||
stats,
|
||||
error,
|
||||
}));
|
||||
|
||||
let elapsed = start_time.elapsed().as_secs();
|
||||
let update_finished_message = format!("🌷 Update process finished! Took {elapsed} secs.");
|
||||
|
||||
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent {
|
||||
target: "Playlist Update".to_string(),
|
||||
message: update_finished_message.clone(),
|
||||
}));
|
||||
events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_global(
|
||||
run_id,
|
||||
execution_order,
|
||||
"Playlist Update",
|
||||
update_finished_message.clone(),
|
||||
)));
|
||||
log_memory_snapshot("exec_processing before_interner_gc");
|
||||
debug!("StringInterner GC removed {} strings", interner_gc());
|
||||
log_memory_snapshot("exec_processing after_interner_gc");
|
||||
@@ -397,5 +489,24 @@ pub async fn exec_processing<E: EventSink + Clone + 'static, B: UpdateBootstrap,
|
||||
info!("{update_finished_message}");
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub(crate) struct PlaylistRunSignals {
|
||||
pub(crate) has_error: bool,
|
||||
pub(crate) has_pending_stalker_refresh: bool,
|
||||
pub(crate) has_quality_rejections: bool,
|
||||
}
|
||||
|
||||
impl PlaylistRunSignals {
|
||||
pub(crate) const fn state(self) -> PlaylistUpdateState {
|
||||
if self.has_error {
|
||||
PlaylistUpdateState::Failure
|
||||
} else if self.has_pending_stalker_refresh || self.has_quality_rejections {
|
||||
PlaylistUpdateState::Partial
|
||||
} else {
|
||||
PlaylistUpdateState::Success
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
mod staged_completion {
|
||||
use super::*;
|
||||
|
||||
pub(super) fn input_completions(events: &CollectSink, input_id: u16) -> Vec<PlaylistUpdateProgressEvent> {
|
||||
events
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
EventMessage::PlaylistUpdateProgress(progress)
|
||||
if progress.input_id == Some(input_id) && progress.state.is_some() =>
|
||||
{
|
||||
Some(progress.clone())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn staged_run_context(
|
||||
root: &Path,
|
||||
base_url: &str,
|
||||
) -> (PlaylistProcessingContext<CollectSink>, Arc<ConfigInput>) {
|
||||
let ctx = processing_context_with_events(root, CollectSink::default());
|
||||
let parent_file = root.join("parent.m3u");
|
||||
std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap();
|
||||
let parent = Arc::new(ConfigInput {
|
||||
id: 7,
|
||||
name: "parent".intern(),
|
||||
input_type: InputType::M3u,
|
||||
url: parent_file.to_string_lossy().into_owned(),
|
||||
enabled: true,
|
||||
..ConfigInput::default()
|
||||
});
|
||||
let mut staged = (*test_input(
|
||||
base_url,
|
||||
ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 },
|
||||
0,
|
||||
&[XtreamCluster::Video, XtreamCluster::Series],
|
||||
))
|
||||
.clone();
|
||||
staged.id = 8;
|
||||
staged.name = "staged".intern();
|
||||
staged.input_type = InputType::Staged;
|
||||
staged.staged_type = shared::model::StagedInputType::Xtream;
|
||||
staged.staged = Some(tuliprox_core::model::ConfigInputStaged {
|
||||
for_input: Some(parent.name.clone()),
|
||||
clusters: ClusterFlags::Live,
|
||||
});
|
||||
staged.resolve_staged_download_type();
|
||||
let staged = Arc::new(staged);
|
||||
seed_live_baseline(&ctx, &staged, 100).await;
|
||||
let failed_companion = Arc::new(ConfigInput {
|
||||
id: 9,
|
||||
name: "failed-companion".intern(),
|
||||
input_type: InputType::M3u,
|
||||
url: root.join("missing.m3u").to_string_lossy().into_owned(),
|
||||
enabled: true,
|
||||
..ConfigInput::default()
|
||||
});
|
||||
ctx.config.sources.store(Arc::new(SourcesConfig {
|
||||
inputs: vec![parent.clone(), staged.clone(), failed_companion.clone()],
|
||||
sources: vec![ConfigSource {
|
||||
inputs: vec![parent.name.clone(), failed_companion.name.clone()],
|
||||
targets: vec![xtream_target(true)],
|
||||
}],
|
||||
..SourcesConfig::default()
|
||||
}));
|
||||
(ctx, staged)
|
||||
}
|
||||
|
||||
fn normalized_trace(
|
||||
events: &[EventMessage],
|
||||
persisted: shared::model::PersistedPlaylistUpdateInputResult,
|
||||
) -> serde_json::Value {
|
||||
let summary = events
|
||||
.iter()
|
||||
.find_map(|event| match event {
|
||||
EventMessage::PlaylistUpdate(summary) => Some(summary),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(summary.run_id, Some("staged-completion-run".into()));
|
||||
assert!(summary.execution_order.is_some());
|
||||
assert_eq!(summary.state, PlaylistUpdateState::Failure);
|
||||
let progress: Vec<_> = events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
EventMessage::PlaylistUpdateProgress(progress)
|
||||
if progress.input_id == Some(8) || progress.state.is_some() =>
|
||||
{
|
||||
assert_eq!(progress.run_id, summary.run_id);
|
||||
assert_eq!(progress.execution_order, summary.execution_order);
|
||||
let mut progress = progress.clone();
|
||||
// Only the globally allocated test-run order is normalized. All
|
||||
// input identities, states, messages and telemetry remain exact.
|
||||
progress.execution_order = Some(1.into());
|
||||
Some(progress)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert!(persisted.timestamp > 0);
|
||||
serde_json::json!({
|
||||
"progress": progress,
|
||||
"completed": {"run_id": summary.run_id, "execution_order": 1, "state": summary.state},
|
||||
"persisted_input_result": {"state": persisted.state, "timestamp": 1}
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_update_status_staged_reporter_trace_matches_persistence_before_foreign_run_failure() {
|
||||
let mut traces = Vec::new();
|
||||
for expected in [PlaylistUpdateState::Partial, PlaylistUpdateState::Success, PlaylistUpdateState::Failure] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut responses =
|
||||
fixture_responses([if expected == PlaylistUpdateState::Success { 100 } else { 1 }, 0, 0]);
|
||||
if expected == PlaylistUpdateState::Failure {
|
||||
responses.insert("get_live_streams".into(), "not JSON".into());
|
||||
}
|
||||
let server = TestXtreamServer::start_with_responses(responses);
|
||||
let (ctx, staged) = staged_run_context(temp.path(), &server.base_url).await;
|
||||
exec_processing(ProcessingRun::for_run(
|
||||
"staged-completion-run".into(),
|
||||
ctx.client.clone(),
|
||||
ctx.config.clone(),
|
||||
ctx.user_targets.clone(),
|
||||
ctx.events.clone(),
|
||||
))
|
||||
.await;
|
||||
let path = input_storage_path(&ctx, &staged).await;
|
||||
let persisted = input_cache::load_input_status(&path);
|
||||
assert!(!persisted.clusters.contains_key("default"));
|
||||
assert_eq!(persisted.last_input_update.unwrap().state, expected);
|
||||
let completions = input_completions(&ctx.events, staged.id);
|
||||
assert_eq!(completions.len(), 1, "real indirect staged input must emit one correlated completion");
|
||||
assert_eq!(completions[0].state, Some(expected));
|
||||
let telemetry = completions[0].input_telemetry.as_ref().unwrap();
|
||||
assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL);
|
||||
assert_eq!(telemetry.source, None);
|
||||
assert_eq!(telemetry.clusters[0].source, Some(PlaylistUpdateDataSource::Provider));
|
||||
assert_eq!(telemetry.clusters[0].threshold, Some(95));
|
||||
if expected == PlaylistUpdateState::Partial {
|
||||
assert_eq!(telemetry.clusters[0].decision, Some(PlaylistUpdateClusterDecision::Rejected));
|
||||
assert_eq!(telemetry.clusters[0].active_count, Some(100));
|
||||
}
|
||||
let emitted = ctx.events.0.lock().unwrap().clone();
|
||||
let own_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(8) && p.state.is_some())).unwrap();
|
||||
let parent_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(7) && p.state.is_some())).unwrap();
|
||||
let completed_index =
|
||||
emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdate(_))).unwrap();
|
||||
assert!(own_index < parent_index && parent_index < completed_index);
|
||||
assert_eq!(input_completions(&ctx.events, 9)[0].state, Some(PlaylistUpdateState::Failure));
|
||||
traces.push(normalized_trace(&emitted, persisted.last_input_update.unwrap()));
|
||||
assert_eq!(server.finish().iter().filter(|action| action.as_str() == "get_live_streams").count(), 1);
|
||||
}
|
||||
let actual = serde_json::Value::Array(traces);
|
||||
let fixture: serde_json::Value = serde_json::from_str(include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../frontend/src/model/test_data/staged_completions.json"
|
||||
)))
|
||||
.unwrap();
|
||||
assert_eq!(actual, fixture, "actual reporter trace: {}", serde_json::to_string_pretty(&actual).unwrap());
|
||||
}
|
||||
}
|
||||
@@ -93,14 +93,23 @@ pub(crate) async fn process_targets<E: EventSink + Clone + 'static, M: MetadataU
|
||||
targets: &[&Arc<ConfigTarget>],
|
||||
input_stats: &mut HashMap<Arc<str>, InputStats>,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
accepted_empty_clusters: ClusterFlags,
|
||||
process_parallel: bool,
|
||||
) -> Vec<TargetStats> {
|
||||
if !process_parallel {
|
||||
let mut target_stats = Vec::with_capacity(targets.len());
|
||||
for (index, target) in targets.iter().enumerate() {
|
||||
let consume_input_source = index + 1 == targets.len();
|
||||
let result =
|
||||
prepare_playlist_for_target(ctx, playlists, target, input_stats, errors, consume_input_source).await;
|
||||
let result = prepare_playlist_for_target(
|
||||
ctx,
|
||||
playlists,
|
||||
target,
|
||||
input_stats,
|
||||
errors,
|
||||
accepted_empty_clusters,
|
||||
consume_input_source,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(prepared) => {
|
||||
let processing = prepared.processing.clone();
|
||||
@@ -142,7 +151,9 @@ pub(crate) async fn process_targets<E: EventSink + Clone + 'static, M: MetadataU
|
||||
let (completion, receiver) = watch::channel(false);
|
||||
completion_receivers.push(receiver);
|
||||
|
||||
match prepare_playlist_for_target(ctx, playlists, target, input_stats, errors, false).await {
|
||||
match prepare_playlist_for_target(ctx, playlists, target, input_stats, errors, accepted_empty_clusters, false)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => {
|
||||
let processing = prepared.processing.clone();
|
||||
let task_ctx = Arc::clone(ctx);
|
||||
@@ -232,10 +243,12 @@ pub(crate) fn apply_persist_filter(target: &ConfigTarget, groups: &mut Vec<Playl
|
||||
}
|
||||
|
||||
pub(crate) struct PreparedTarget {
|
||||
pub(crate) library_empty: tuliprox_repository::LibraryEmptyPublication,
|
||||
pub(crate) target: ConfigTarget,
|
||||
pub(crate) playlist: Vec<PlaylistGroup>,
|
||||
pub(crate) epg: Vec<Epg>,
|
||||
pub(crate) processing: PipelineStats,
|
||||
pub(crate) accepted_empty_clusters: ClusterFlags,
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -245,6 +258,7 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
|
||||
target: &ConfigTarget,
|
||||
stats: &mut HashMap<Arc<str>, InputStats>,
|
||||
errors: &mut Vec<TuliproxError>,
|
||||
accepted_empty_clusters: ClusterFlags,
|
||||
consume_input_source: bool,
|
||||
) -> Result<PreparedTarget, Vec<TuliproxError>> {
|
||||
debug_if_enabled!("Processing order is {}", &target.processing_order);
|
||||
@@ -254,9 +268,15 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
|
||||
let mut new_epg = vec![];
|
||||
let mut new_playlist: Vec<PlaylistGroup> = vec![];
|
||||
let mut aggregate_outcome = PipelineOutcome::default();
|
||||
let mut has_empty_library = false;
|
||||
let mut all_inputs_empty = !playlists.is_empty();
|
||||
let mut only_library_inputs = true;
|
||||
// The caller has already required Ready for every input. An empty foreign input
|
||||
// still needs its own Force contract; it cannot borrow Library's filter permission.
|
||||
let mut non_library_inputs_populated = true;
|
||||
|
||||
debug!("Executing processing pipes");
|
||||
let broadcast_step = create_broadcast_callback(&ctx.events);
|
||||
let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order);
|
||||
|
||||
let bouquet_file =
|
||||
tuliprox_repository::load_target_bouquet(&ctx.config, &target.name).await.map_err(|err| vec![err])?;
|
||||
@@ -270,6 +290,11 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
|
||||
let pipe = get_processing_pipe(target);
|
||||
let mut step = StepMeasure::new(&target.name, broadcast_step);
|
||||
for provider_fpl in playlists.iter_mut() {
|
||||
let input_is_empty = provider_fpl.source.is_empty();
|
||||
all_inputs_empty &= input_is_empty;
|
||||
only_library_inputs &= provider_fpl.input.input_type == InputType::Library;
|
||||
non_library_inputs_populated &= provider_fpl.input.input_type == InputType::Library || !input_is_empty;
|
||||
has_empty_library |= provider_fpl.input.input_type == InputType::Library && input_is_empty;
|
||||
log_memory_snapshot(
|
||||
format!("target '{}' input '{}' before_pipe", target.name, provider_fpl.input.name).as_str(),
|
||||
);
|
||||
@@ -319,10 +344,19 @@ pub(crate) async fn prepare_playlist_for_target<E: EventSink + Clone + 'static,
|
||||
log_memory_snapshot(format!("target '{}' after_filter_rename_map_epg", target.name).as_str());
|
||||
step.stop("Preparing playlist");
|
||||
Ok(PreparedTarget {
|
||||
library_empty: match (has_empty_library, all_inputs_empty && only_library_inputs) {
|
||||
(true, true) => tuliprox_repository::LibraryEmptyPublication::CompleteTarget,
|
||||
(true, false) if non_library_inputs_populated && accepted_empty_clusters.is_empty() => {
|
||||
tuliprox_repository::LibraryEmptyPublication::FilterableContribution
|
||||
}
|
||||
(true, false) => tuliprox_repository::LibraryEmptyPublication::Contribution,
|
||||
(false, _) => tuliprox_repository::LibraryEmptyPublication::None,
|
||||
},
|
||||
target: target.clone(),
|
||||
playlist: new_playlist,
|
||||
epg: new_epg,
|
||||
processing: aggregate_outcome.to_stats(),
|
||||
accepted_empty_clusters,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -373,7 +407,7 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
|
||||
let mut new_playlist = prepared.playlist;
|
||||
let mut new_epg = prepared.epg;
|
||||
let mut errors = Vec::new();
|
||||
let broadcast_step = create_broadcast_callback(&ctx.events);
|
||||
let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order);
|
||||
let mut step = StepMeasure::new(&target.name, broadcast_step);
|
||||
if target.favourites.is_some() {
|
||||
step.broadcast("Processing favourites for '{}' playlist", &target.name);
|
||||
@@ -381,7 +415,9 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
|
||||
log_memory_snapshot(format!("target '{}' after_favourites", target.name).as_str());
|
||||
}
|
||||
|
||||
if new_playlist.is_empty() {
|
||||
let library_empty = prepared.library_empty.for_filtered_playlist(&new_playlist);
|
||||
if new_playlist.is_empty() && prepared.accepted_empty_clusters.is_empty() && !library_empty.replaces_empty_target()
|
||||
{
|
||||
step.stop("");
|
||||
info!("Playlist is empty: {}", target.name);
|
||||
(Ok(()), errors)
|
||||
@@ -455,12 +491,14 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
|
||||
} else {
|
||||
flatten_tvguide(new_epg)
|
||||
};
|
||||
let library_empty = library_empty.for_filtered_playlist(&flat_new_playlist);
|
||||
let result = persist_playlist(
|
||||
&ctx.config,
|
||||
&mut flat_new_playlist,
|
||||
merged_epg.as_ref(),
|
||||
target,
|
||||
ctx.playlist_state.as_ref(),
|
||||
TargetPlaylistPersistOptions { accepted_empty_clusters: prepared.accepted_empty_clusters, library_empty },
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() && process_watch(&ctx.config, &ctx.events, target, &flat_new_playlist).await {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -41,16 +41,16 @@ impl PlaylistProvider for StalkerProvider {
|
||||
fn name(&self) -> &'static str { "stalker" }
|
||||
|
||||
async fn fetch(&self, request: &PlaylistFetchRequest<'_>) -> PlaylistFetch {
|
||||
let (groups, errors, persisted, partial) = download_stalker_playlist(
|
||||
download_stalker_playlist(
|
||||
request.app_config,
|
||||
request.client,
|
||||
request.input,
|
||||
None,
|
||||
self.refresh_mode,
|
||||
self.materialize_active,
|
||||
request.update_quality,
|
||||
)
|
||||
.await;
|
||||
PlaylistFetch::groups(groups).with_errors(errors).persisted(persisted).partial(partial)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,15 +10,15 @@
|
||||
|
||||
#![allow(clippy::too_many_lines, clippy::needless_pass_by_value)]
|
||||
|
||||
use super::stalker_refresh::{
|
||||
advance_stalker_refresh, StalkerClusterSelection, StalkerRefreshMode, StalkerRefreshOutcome,
|
||||
};
|
||||
use super::stalker_refresh::{advance_stalker_refresh, StalkerRefreshMode, StalkerRefreshOutcome, StalkerRefreshPlan};
|
||||
use log::{debug, info, warn};
|
||||
use lru::LruCache;
|
||||
use parking_lot::Mutex;
|
||||
use shared::{
|
||||
error::TuliproxError,
|
||||
model::{stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, PlaylistGroup, PlaylistItem},
|
||||
model::{
|
||||
stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, PlaylistGroup, PlaylistItem, UpdateQualityPolicy,
|
||||
},
|
||||
utils::Internable,
|
||||
};
|
||||
use std::{
|
||||
@@ -29,7 +29,10 @@ use std::{
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tuliprox_core::model::{AppConfig, ConfigInput, ConfigInputFlags, StalkerInputConfig};
|
||||
use tuliprox_iptv::stalker::{client::StalkerApiClient, error::StalkerError, parser};
|
||||
use tuliprox_iptv::{
|
||||
provider::PlaylistFetch,
|
||||
stalker::{client::StalkerApiClient, error::StalkerError, parser},
|
||||
};
|
||||
use tuliprox_repository::{
|
||||
stalker_generation_repository::{load_active_manifest, load_checkpoint},
|
||||
stalker_repository::{ensure_stalker_storage_path, load_stalker_items_at, read_stalker_item_at},
|
||||
@@ -87,6 +90,24 @@ async fn try_acquire_stalker_refresh(input_name: &Arc<str>) -> Option<tokio::syn
|
||||
semaphore.try_acquire_owned().ok()
|
||||
}
|
||||
|
||||
fn stalker_refresh_busy_fetch(input_name: &str, persisted: bool, quality_policy: UpdateQualityPolicy) -> PlaylistFetch {
|
||||
if matches!(quality_policy, UpdateQualityPolicy::Bypass) {
|
||||
PlaylistFetch::failed(TuliproxError::ProviderConnection(format!(
|
||||
"Stalker refresh for input '{input_name}' is busy; retry the forced update"
|
||||
)))
|
||||
.persisted(persisted)
|
||||
} else {
|
||||
PlaylistFetch::nothing_to_do().persisted(persisted).partial(true)
|
||||
}
|
||||
}
|
||||
|
||||
fn failed_stalker_acquisition(error: TuliproxError, clusters: &[StalkerCluster]) -> PlaylistFetch {
|
||||
PlaylistFetch {
|
||||
failed_clusters: clusters.iter().copied().map(xtream_cluster).collect(),
|
||||
..PlaylistFetch::failed(error)
|
||||
}
|
||||
}
|
||||
|
||||
fn cached_resolved_link(key: RuntimeLinkKey, force_refresh: bool) -> Option<Arc<str>> {
|
||||
let mut cache = RUNTIME_STALKER_LINKS.lock();
|
||||
let expired = cache.peek(&key).is_some_and(|entry| entry.expires_at <= Instant::now());
|
||||
@@ -111,19 +132,15 @@ pub async fn download_stalker_playlist(
|
||||
clusters: Option<&[StalkerCluster]>,
|
||||
refresh_mode: StalkerRefreshMode,
|
||||
materialize_active: bool,
|
||||
) -> (Vec<PlaylistGroup>, Vec<TuliproxError>, bool, bool) {
|
||||
quality_policy: UpdateQualityPolicy,
|
||||
) -> PlaylistFetch {
|
||||
let stalker_cfg = match input.stalker.as_ref() {
|
||||
Some(cfg) => cfg.clone(),
|
||||
None => {
|
||||
return (
|
||||
vec![],
|
||||
vec![TuliproxError::ConfigInput(format!(
|
||||
"Stalker input '{}' has no stalker configuration block",
|
||||
input.name
|
||||
))],
|
||||
false,
|
||||
false,
|
||||
);
|
||||
return PlaylistFetch::failed(TuliproxError::ConfigInput(format!(
|
||||
"Stalker input '{}' has no stalker configuration block",
|
||||
input.name
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -136,27 +153,22 @@ pub async fn download_stalker_playlist(
|
||||
|
||||
if resolved_clusters.is_empty() {
|
||||
info!("Stalker input '{}' has all clusters skipped", input.name);
|
||||
return (vec![], vec![], false, false);
|
||||
return PlaylistFetch::nothing_to_do();
|
||||
}
|
||||
let refresh_selection = StalkerClusterSelection::requested(input, &resolved_clusters);
|
||||
let refresh_plan = StalkerRefreshPlan::requested(input, &resolved_clusters, quality_policy);
|
||||
|
||||
let portal_url = match resolve_stalker_portal_url(input) {
|
||||
Ok(url) => url,
|
||||
Err(err) => return (vec![], vec![err], false, false),
|
||||
Err(err) => return failed_stalker_acquisition(err, &resolved_clusters),
|
||||
};
|
||||
|
||||
let identity_fingerprint = stalker_identity_fingerprint(&portal_url, &stalker_cfg);
|
||||
let api_client = match cached_runtime_stalker_client(client, portal_url, &stalker_cfg) {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
return (
|
||||
vec![],
|
||||
vec![TuliproxError::ConfigInput(format!(
|
||||
"failed to build Stalker client for input '{}': {err}",
|
||||
input.name
|
||||
))],
|
||||
false,
|
||||
false,
|
||||
return failed_stalker_acquisition(
|
||||
TuliproxError::ConfigInput(format!("failed to build Stalker client for input '{}': {err}", input.name)),
|
||||
&resolved_clusters,
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -164,11 +176,9 @@ pub async fn download_stalker_playlist(
|
||||
let storage_path = match ensure_stalker_storage_path(app_config, &input.name).await {
|
||||
Ok(p) => p,
|
||||
Err(err) => {
|
||||
return (
|
||||
vec![],
|
||||
vec![TuliproxError::Io(format!("could not prepare Stalker storage for input '{}': {err}", input.name))],
|
||||
false,
|
||||
false,
|
||||
return failed_stalker_acquisition(
|
||||
TuliproxError::Io(format!("could not prepare Stalker storage for input '{}': {err}", input.name)),
|
||||
&resolved_clusters,
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -178,14 +188,12 @@ pub async fn download_stalker_playlist(
|
||||
let handshake = match api_client.handshake().await {
|
||||
Ok(handshake) => handshake,
|
||||
Err(err) => {
|
||||
return (
|
||||
vec![],
|
||||
vec![TuliproxError::ProviderConnection(format!(
|
||||
return failed_stalker_acquisition(
|
||||
TuliproxError::ProviderConnection(format!(
|
||||
"Stalker handshake for input '{}' failed: {err}",
|
||||
input.name
|
||||
))],
|
||||
false,
|
||||
false,
|
||||
)),
|
||||
&resolved_clusters,
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -194,7 +202,7 @@ pub async fn download_stalker_playlist(
|
||||
app_config,
|
||||
api_client.as_ref(),
|
||||
&handshake,
|
||||
refresh_selection,
|
||||
refresh_plan,
|
||||
&storage_path,
|
||||
identity_fingerprint,
|
||||
refresh_mode.budget(),
|
||||
@@ -205,7 +213,7 @@ pub async fn download_stalker_playlist(
|
||||
} else {
|
||||
let checkpoint = match load_checkpoint(&storage_path, identity_fingerprint).await {
|
||||
Ok(checkpoint) => checkpoint,
|
||||
Err(err) => return (Vec::new(), vec![err], false, false),
|
||||
Err(err) => return PlaylistFetch::failed(err),
|
||||
};
|
||||
break StalkerRefreshOutcome::Yielded {
|
||||
phase: checkpoint.as_ref().map_or(
|
||||
@@ -227,18 +235,40 @@ pub async fn download_stalker_playlist(
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
Ok(outcome) => break outcome,
|
||||
Err(err) => return (Vec::new(), vec![err], false, false),
|
||||
Err(err) => return PlaylistFetch::failed(err),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return (Vec::new(), Vec::new(), app_config.config.load().disk_based_processing, true);
|
||||
return stalker_refresh_busy_fetch(&input.name, app_config.config.load().disk_based_processing, quality_policy);
|
||||
};
|
||||
|
||||
let yielded = matches!(&outcome, StalkerRefreshOutcome::Yielded { .. });
|
||||
let terminal = matches!(&outcome, StalkerRefreshOutcome::Terminal(_));
|
||||
let mut errors = Vec::new();
|
||||
let mut quality_acceptances = Vec::new();
|
||||
let mut quality_rejections = Vec::new();
|
||||
let mut force_updates = Vec::new();
|
||||
match outcome {
|
||||
StalkerRefreshOutcome::Complete => {}
|
||||
StalkerRefreshOutcome::Complete {
|
||||
quality_acceptances: completed_acceptances,
|
||||
quality_rejections: completed_rejections,
|
||||
force_updates: completed_force_updates,
|
||||
} => {
|
||||
quality_acceptances = completed_acceptances;
|
||||
quality_rejections = completed_rejections;
|
||||
force_updates = completed_force_updates;
|
||||
}
|
||||
StalkerRefreshOutcome::Failed {
|
||||
quality_acceptances: completed_acceptances,
|
||||
quality_rejections: completed_rejections,
|
||||
force_updates: completed_force_updates,
|
||||
error,
|
||||
} => {
|
||||
return PlaylistFetch::failed(error)
|
||||
.with_quality_acceptances(completed_acceptances)
|
||||
.with_quality_rejections(completed_rejections)
|
||||
.with_force_updates(completed_force_updates);
|
||||
}
|
||||
StalkerRefreshOutcome::Yielded { phase, processed, skipped, error } => {
|
||||
info!(
|
||||
"Stalker input '{}' yielded in phase {phase:?} after {processed} records ({skipped} skipped)",
|
||||
@@ -255,7 +285,12 @@ pub async fn download_stalker_playlist(
|
||||
Ok(manifest) => manifest,
|
||||
Err(err) => {
|
||||
errors.push(err);
|
||||
return (Vec::new(), errors, false, yielded);
|
||||
return PlaylistFetch::groups(Vec::new())
|
||||
.with_errors(errors)
|
||||
.with_quality_acceptances(quality_acceptances)
|
||||
.with_quality_rejections(quality_rejections)
|
||||
.with_force_updates(force_updates)
|
||||
.partial(yielded);
|
||||
}
|
||||
};
|
||||
if terminal {
|
||||
@@ -268,7 +303,13 @@ pub async fn download_stalker_playlist(
|
||||
}
|
||||
let use_disk_based_processing = app_config.config.load().disk_based_processing;
|
||||
if !materialize_active {
|
||||
return (Vec::new(), errors, use_disk_based_processing, yielded);
|
||||
return PlaylistFetch::groups(Vec::new())
|
||||
.with_errors(errors)
|
||||
.with_quality_acceptances(quality_acceptances)
|
||||
.with_quality_rejections(quality_rejections)
|
||||
.with_force_updates(force_updates)
|
||||
.persisted(use_disk_based_processing)
|
||||
.partial(yielded);
|
||||
}
|
||||
let mut groups = Vec::new();
|
||||
let mut counts = [0_usize; 3];
|
||||
@@ -324,7 +365,13 @@ pub async fn download_stalker_playlist(
|
||||
}
|
||||
}
|
||||
parser::log_stalker_download_summary(&input.name, counts[0], counts[1], counts[2]);
|
||||
(groups, errors, use_disk_based_processing, yielded)
|
||||
PlaylistFetch::groups(groups)
|
||||
.with_errors(errors)
|
||||
.with_quality_acceptances(quality_acceptances)
|
||||
.with_quality_rejections(quality_rejections)
|
||||
.with_force_updates(force_updates)
|
||||
.persisted(use_disk_based_processing)
|
||||
.partial(yielded)
|
||||
}
|
||||
|
||||
/// Resolve a per-input portal URL using the standard `provider://` resolution pipeline.
|
||||
@@ -584,4 +631,31 @@ mod tests {
|
||||
drop(first);
|
||||
assert!(try_acquire_stalker_refresh(&name).await.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_stalker_refresh_busy_is_visible_and_does_not_mutate_checkpoint() {
|
||||
use tuliprox_repository::stalker_generation_repository::{save_checkpoint, StalkerCheckpoint};
|
||||
|
||||
let temp = tempfile::tempdir().expect("temporary Stalker storage");
|
||||
let input_name: Arc<str> = "force-busy-portal".into();
|
||||
let identity = 17;
|
||||
save_checkpoint(temp.path(), &StalkerCheckpoint::new(identity, 23, 0b1111, 123))
|
||||
.await
|
||||
.expect("checkpoint should persist");
|
||||
|
||||
let refresh_permit = try_acquire_stalker_refresh(&input_name).await.expect("first refresh owns permit");
|
||||
let fetch = stalker_refresh_busy_fetch(&input_name, true, UpdateQualityPolicy::Bypass);
|
||||
|
||||
assert_eq!(fetch.errors.len(), 1);
|
||||
assert!(!fetch.partial);
|
||||
assert!(fetch.persisted);
|
||||
let checkpoint = load_checkpoint(temp.path(), identity)
|
||||
.await
|
||||
.expect("checkpoint should load")
|
||||
.expect("checkpoint should remain present");
|
||||
assert_eq!(checkpoint.quality_bypass_mask, 0);
|
||||
|
||||
drop(refresh_permit);
|
||||
assert!(try_acquire_stalker_refresh(&input_name).await.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::stalker::StalkerCluster;
|
||||
use shared::{
|
||||
error::TuliproxError,
|
||||
model::{stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem},
|
||||
model::{stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, UpdateQualityPolicy, XtreamCluster},
|
||||
};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
@@ -9,7 +9,10 @@ use std::{
|
||||
sync::Arc,
|
||||
time::{Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use tuliprox_core::model::{AppConfig, ConfigInput, ConfigInputFlags};
|
||||
use tuliprox_core::model::{
|
||||
evaluate_update_quality, AppConfig, ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection,
|
||||
ConfigInput, ConfigInputFlags, ConfigInputUpdateQuality,
|
||||
};
|
||||
use tuliprox_iptv::stalker::{
|
||||
catalog::{StalkerCategory, StalkerRawItem},
|
||||
client::StalkerApiClient,
|
||||
@@ -23,13 +26,19 @@ use tuliprox_repository::{
|
||||
publish_selection, save_checkpoint, StalkerCheckpoint, StalkerGenerationData, StalkerRefreshPhase,
|
||||
},
|
||||
stalker_repository::{
|
||||
load_stalker_items_after, prepare_stalker_episode_series_at, promote_stalker_file, remove_stalker_file,
|
||||
snapshot_stalker_epg_at, snapshot_stalker_items_at, upsert_stalker_epg_at, upsert_stalker_items_at,
|
||||
count_stalker_items_at, load_stalker_items_after, prepare_stalker_episode_series_at, promote_stalker_file,
|
||||
remove_stalker_file, snapshot_stalker_epg_at, snapshot_stalker_items_at, upsert_stalker_epg_at,
|
||||
upsert_stalker_items_at,
|
||||
},
|
||||
};
|
||||
|
||||
const MAX_RETRIES: u8 = 3;
|
||||
const SKIPPED_SAMPLE_LIMIT: usize = 32;
|
||||
const LIVE_SELECTION: u8 = 0b0001;
|
||||
const VOD_SELECTION: u8 = 0b0010;
|
||||
const SERIES_SELECTION: u8 = 0b0100;
|
||||
const EPG_SELECTION: u8 = 0b1000;
|
||||
const MEDIA_SELECTION: u8 = LIVE_SELECTION | VOD_SELECTION | SERIES_SELECTION;
|
||||
|
||||
pub enum StalkerRefreshLimit {
|
||||
Deadline(Instant),
|
||||
@@ -118,6 +127,26 @@ impl StalkerClusterSelection {
|
||||
}
|
||||
}
|
||||
|
||||
/// Requested clusters and the publication policy applied once their generation is complete.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct StalkerRefreshPlan {
|
||||
selection: StalkerClusterSelection,
|
||||
update_quality: ConfigInputUpdateQuality,
|
||||
quality_bypass_mask: u8,
|
||||
}
|
||||
|
||||
impl StalkerRefreshPlan {
|
||||
pub fn requested(input: &ConfigInput, requested: &[StalkerCluster], quality_policy: UpdateQualityPolicy) -> Self {
|
||||
let selection = StalkerClusterSelection::requested(input, requested);
|
||||
let update_quality =
|
||||
input.options.as_ref().map_or_else(ConfigInputUpdateQuality::default, |options| options.update_quality);
|
||||
let quality_bypass_mask = matches!(quality_policy, UpdateQualityPolicy::Bypass)
|
||||
.then_some(selection.mask() & MEDIA_SELECTION)
|
||||
.unwrap_or(0);
|
||||
Self { selection, update_quality, quality_bypass_mask }
|
||||
}
|
||||
}
|
||||
|
||||
fn first_phase(selection: StalkerClusterSelection) -> StalkerRefreshPhase {
|
||||
if selection.live {
|
||||
StalkerRefreshPhase::LiveBulk
|
||||
@@ -154,8 +183,23 @@ fn next_phase_after_series(selection: StalkerClusterSelection) -> StalkerRefresh
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum StalkerRefreshOutcome {
|
||||
Complete,
|
||||
Yielded { phase: StalkerRefreshPhase, processed: u64, skipped: u64, error: Option<TuliproxError> },
|
||||
Complete {
|
||||
quality_acceptances: Vec<ClusterUpdateAcceptance>,
|
||||
quality_rejections: Vec<ClusterUpdateRejection>,
|
||||
force_updates: Vec<ClusterForceUpdate>,
|
||||
},
|
||||
Failed {
|
||||
quality_acceptances: Vec<ClusterUpdateAcceptance>,
|
||||
quality_rejections: Vec<ClusterUpdateRejection>,
|
||||
force_updates: Vec<ClusterForceUpdate>,
|
||||
error: TuliproxError,
|
||||
},
|
||||
Yielded {
|
||||
phase: StalkerRefreshPhase,
|
||||
processed: u64,
|
||||
skipped: u64,
|
||||
error: Option<TuliproxError>,
|
||||
},
|
||||
Terminal(TuliproxError),
|
||||
}
|
||||
|
||||
@@ -169,9 +213,15 @@ async fn load_or_start_checkpoint(
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
selection: StalkerClusterSelection,
|
||||
quality_bypass_mask: u8,
|
||||
) -> Result<StalkerCheckpoint, TuliproxError> {
|
||||
if let Some(state) = load_checkpoint(storage_path, identity_fingerprint).await? {
|
||||
if let Some(mut state) = load_checkpoint(storage_path, identity_fingerprint).await? {
|
||||
if state.selection_mask == selection.mask() {
|
||||
let combined_bypass_mask = state.quality_bypass_mask | quality_bypass_mask;
|
||||
if combined_bypass_mask != state.quality_bypass_mask {
|
||||
state.quality_bypass_mask = combined_bypass_mask;
|
||||
save_checkpoint(storage_path, &state).await?;
|
||||
}
|
||||
return Ok(state);
|
||||
}
|
||||
}
|
||||
@@ -179,30 +229,70 @@ async fn load_or_start_checkpoint(
|
||||
let mut state =
|
||||
StalkerCheckpoint::new(identity_fingerprint, generation_id(), selection.mask(), chrono::Utc::now().timestamp());
|
||||
state.phase = first_phase(selection);
|
||||
state.quality_bypass_mask = quality_bypass_mask;
|
||||
save_checkpoint(storage_path, &state).await?;
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
async fn finish_completed_refresh_checked(
|
||||
async fn evaluate_completed_refresh_publication(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
checkpoint: &StalkerCheckpoint,
|
||||
update_quality: ConfigInputUpdateQuality,
|
||||
) -> Result<StalkerGenerationPublication, TuliproxError> {
|
||||
let needs_quality_evaluation =
|
||||
selection_needs_quality_evaluation(checkpoint.selection_mask, checkpoint.quality_bypass_mask, update_quality);
|
||||
let has_quality_bypass = checkpoint.quality_bypass_mask & MEDIA_SELECTION != 0;
|
||||
if needs_quality_evaluation || has_quality_bypass {
|
||||
let active_manifest = if needs_quality_evaluation {
|
||||
Some(load_active_manifest(storage_path, identity_fingerprint).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
evaluate_generation_publication(
|
||||
app_config,
|
||||
storage_path,
|
||||
checkpoint.generation,
|
||||
checkpoint.selection_mask,
|
||||
checkpoint.quality_bypass_mask,
|
||||
update_quality,
|
||||
active_manifest.as_ref(),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Ok(StalkerGenerationPublication {
|
||||
accepted_selection_mask: checkpoint.selection_mask,
|
||||
quality_acceptances: Vec::new(),
|
||||
quality_rejections: Vec::new(),
|
||||
force_updates: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_completed_refresh_has_usable_playlist(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
checkpoint: &StalkerCheckpoint,
|
||||
accepted_selection_mask: u8,
|
||||
) -> Result<(), TuliproxError> {
|
||||
if checkpoint.processed == 0 {
|
||||
let accepted_media_selection = accepted_selection_mask & MEDIA_SELECTION;
|
||||
let accepted_enforced_selection = accepted_media_selection & !checkpoint.quality_bypass_mask;
|
||||
if checkpoint.processed == 0 && accepted_enforced_selection != 0 {
|
||||
let active = load_active_manifest(storage_path, identity_fingerprint).await?;
|
||||
let mut retained_has_items = false;
|
||||
if checkpoint.selection_mask & 0b0001 == 0 {
|
||||
if accepted_selection_mask & LIVE_SELECTION == 0 {
|
||||
if let Some(files) = active.live.as_ref() {
|
||||
retained_has_items = !load_stalker_items_after(app_config, &files.data, None, 1).await?.is_empty();
|
||||
}
|
||||
}
|
||||
if !retained_has_items && checkpoint.selection_mask & 0b0010 == 0 {
|
||||
if !retained_has_items && accepted_selection_mask & VOD_SELECTION == 0 {
|
||||
if let Some(files) = active.vod.as_ref() {
|
||||
retained_has_items = !load_stalker_items_after(app_config, &files.data, None, 1).await?.is_empty();
|
||||
}
|
||||
}
|
||||
if !retained_has_items && checkpoint.selection_mask & 0b0100 == 0 {
|
||||
if !retained_has_items && accepted_selection_mask & SERIES_SELECTION == 0 {
|
||||
if let Some(files) = active.series.as_ref() {
|
||||
retained_has_items = !load_stalker_items_after(app_config, &files.roots, None, 1).await?.is_empty()
|
||||
|| !load_stalker_items_after(app_config, &files.episodes, None, 1).await?.is_empty();
|
||||
@@ -216,18 +306,242 @@ async fn finish_completed_refresh_checked(
|
||||
));
|
||||
}
|
||||
}
|
||||
finish_completed_refresh(storage_path, identity_fingerprint, checkpoint).await
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finish_completed_refresh(
|
||||
async fn publish_completed_refresh(
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
checkpoint: &StalkerCheckpoint,
|
||||
accepted_selection_mask: u8,
|
||||
) -> Result<(), TuliproxError> {
|
||||
let manifest =
|
||||
publish_selection(storage_path, identity_fingerprint, checkpoint.generation, checkpoint.selection_mask).await?;
|
||||
publish_selection(storage_path, identity_fingerprint, checkpoint.generation, accepted_selection_mask).await?;
|
||||
clear_checkpoint(storage_path).await?;
|
||||
cleanup_obsolete_generations(storage_path, &manifest).await
|
||||
cleanup_obsolete_generations(storage_path, &manifest).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct StalkerRefreshReports {
|
||||
quality_acceptances: Vec<ClusterUpdateAcceptance>,
|
||||
quality_rejections: Vec<ClusterUpdateRejection>,
|
||||
force_updates: Vec<ClusterForceUpdate>,
|
||||
error: Option<TuliproxError>,
|
||||
}
|
||||
|
||||
impl StalkerRefreshReports {
|
||||
fn into_outcome(self) -> StalkerRefreshOutcome {
|
||||
let Self { quality_acceptances, quality_rejections, force_updates, error } = self;
|
||||
if let Some(error) = error {
|
||||
StalkerRefreshOutcome::Failed { quality_acceptances, quality_rejections, force_updates, error }
|
||||
} else {
|
||||
StalkerRefreshOutcome::Complete { quality_acceptances, quality_rejections, force_updates }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn finish_completed_refresh(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
checkpoint: &StalkerCheckpoint,
|
||||
update_quality: ConfigInputUpdateQuality,
|
||||
) -> Result<Vec<ClusterUpdateRejection>, TuliproxError> {
|
||||
let publication = evaluate_completed_refresh_publication(
|
||||
app_config,
|
||||
storage_path,
|
||||
identity_fingerprint,
|
||||
checkpoint,
|
||||
update_quality,
|
||||
)
|
||||
.await?;
|
||||
let StalkerGenerationPublication {
|
||||
accepted_selection_mask,
|
||||
quality_rejections,
|
||||
quality_acceptances: _,
|
||||
force_updates: _,
|
||||
} = publication;
|
||||
publish_completed_refresh(storage_path, identity_fingerprint, checkpoint, accepted_selection_mask).await?;
|
||||
Ok(quality_rejections)
|
||||
}
|
||||
|
||||
async fn finish_completed_refresh_checked(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
checkpoint: &StalkerCheckpoint,
|
||||
update_quality: ConfigInputUpdateQuality,
|
||||
) -> Result<StalkerRefreshReports, TuliproxError> {
|
||||
let publication = evaluate_completed_refresh_publication(
|
||||
app_config,
|
||||
storage_path,
|
||||
identity_fingerprint,
|
||||
checkpoint,
|
||||
update_quality,
|
||||
)
|
||||
.await?;
|
||||
let StalkerGenerationPublication {
|
||||
accepted_selection_mask,
|
||||
quality_acceptances,
|
||||
quality_rejections,
|
||||
force_updates,
|
||||
} = publication;
|
||||
let completion_result = async {
|
||||
ensure_completed_refresh_has_usable_playlist(
|
||||
app_config,
|
||||
storage_path,
|
||||
identity_fingerprint,
|
||||
checkpoint,
|
||||
accepted_selection_mask,
|
||||
)
|
||||
.await?;
|
||||
publish_completed_refresh(storage_path, identity_fingerprint, checkpoint, accepted_selection_mask).await
|
||||
}
|
||||
.await;
|
||||
Ok(StalkerRefreshReports { quality_acceptances, quality_rejections, force_updates, error: completion_result.err() })
|
||||
}
|
||||
|
||||
struct StalkerGenerationPublication {
|
||||
accepted_selection_mask: u8,
|
||||
quality_acceptances: Vec<ClusterUpdateAcceptance>,
|
||||
quality_rejections: Vec<ClusterUpdateRejection>,
|
||||
force_updates: Vec<ClusterForceUpdate>,
|
||||
}
|
||||
|
||||
fn selection_needs_quality_evaluation(
|
||||
requested_selection_mask: u8,
|
||||
quality_bypass_mask: u8,
|
||||
update_quality: ConfigInputUpdateQuality,
|
||||
) -> bool {
|
||||
[StalkerCluster::Live, StalkerCluster::Vod, StalkerCluster::Series].into_iter().any(|cluster| {
|
||||
let bit = selection_bit(cluster);
|
||||
requested_selection_mask & bit != 0
|
||||
&& quality_bypass_mask & bit == 0
|
||||
&& update_quality.threshold(quality_cluster(cluster)) != 0
|
||||
})
|
||||
}
|
||||
|
||||
const fn selection_bit(cluster: StalkerCluster) -> u8 {
|
||||
match cluster {
|
||||
StalkerCluster::Live => LIVE_SELECTION,
|
||||
StalkerCluster::Vod => VOD_SELECTION,
|
||||
StalkerCluster::Series => SERIES_SELECTION,
|
||||
}
|
||||
}
|
||||
|
||||
const fn quality_cluster(cluster: StalkerCluster) -> XtreamCluster {
|
||||
match cluster {
|
||||
StalkerCluster::Live => XtreamCluster::Live,
|
||||
StalkerCluster::Vod => XtreamCluster::Video,
|
||||
StalkerCluster::Series => XtreamCluster::Series,
|
||||
}
|
||||
}
|
||||
|
||||
async fn count_stalker_cluster_files(
|
||||
app_config: &Arc<AppConfig>,
|
||||
primary: &Path,
|
||||
secondary: Option<&Path>,
|
||||
) -> Result<usize, TuliproxError> {
|
||||
let primary_count = count_stalker_items_at(app_config, primary).await?.unwrap_or_default();
|
||||
let secondary_count = match secondary {
|
||||
Some(path) => count_stalker_items_at(app_config, path).await?.unwrap_or_default(),
|
||||
None => 0,
|
||||
};
|
||||
primary_count.checked_add(secondary_count).ok_or_else(|| {
|
||||
TuliproxError::RepositoryStalker("Stalker cluster item count exceeds platform capacity".to_string())
|
||||
})
|
||||
}
|
||||
|
||||
async fn count_candidate_cluster(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
generation: u64,
|
||||
cluster: StalkerCluster,
|
||||
) -> Result<usize, TuliproxError> {
|
||||
let primary_data = match cluster {
|
||||
StalkerCluster::Live => StalkerGenerationData::Live,
|
||||
StalkerCluster::Vod => StalkerGenerationData::Vod,
|
||||
StalkerCluster::Series => StalkerGenerationData::SeriesRoots,
|
||||
};
|
||||
let primary = generation_data_path(storage_path, generation, primary_data);
|
||||
let secondary = (cluster == StalkerCluster::Series)
|
||||
.then(|| generation_data_path(storage_path, generation, StalkerGenerationData::SeriesEpisodes));
|
||||
count_stalker_cluster_files(app_config, &primary, secondary.as_deref()).await
|
||||
}
|
||||
|
||||
async fn count_active_cluster(
|
||||
app_config: &Arc<AppConfig>,
|
||||
manifest: &tuliprox_repository::stalker_generation_repository::StalkerActiveManifest,
|
||||
cluster: StalkerCluster,
|
||||
) -> Result<Option<usize>, TuliproxError> {
|
||||
match cluster {
|
||||
StalkerCluster::Live => match manifest.live.as_ref() {
|
||||
Some(files) => count_stalker_cluster_files(app_config, &files.data, None).await.map(Some),
|
||||
None => Ok(None),
|
||||
},
|
||||
StalkerCluster::Vod => match manifest.vod.as_ref() {
|
||||
Some(files) => count_stalker_cluster_files(app_config, &files.data, None).await.map(Some),
|
||||
None => Ok(None),
|
||||
},
|
||||
StalkerCluster::Series => match manifest.series.as_ref() {
|
||||
Some(files) => count_stalker_cluster_files(app_config, &files.roots, Some(&files.episodes)).await.map(Some),
|
||||
None => Ok(None),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn evaluate_generation_publication(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
generation: u64,
|
||||
requested_selection_mask: u8,
|
||||
quality_bypass_mask: u8,
|
||||
update_quality: ConfigInputUpdateQuality,
|
||||
active_manifest: Option<&tuliprox_repository::stalker_generation_repository::StalkerActiveManifest>,
|
||||
) -> Result<StalkerGenerationPublication, TuliproxError> {
|
||||
let mut accepted_selection_mask = requested_selection_mask;
|
||||
let mut quality_acceptances = Vec::new();
|
||||
let mut quality_rejections = Vec::new();
|
||||
let mut force_updates = Vec::new();
|
||||
for cluster in [StalkerCluster::Live, StalkerCluster::Vod, StalkerCluster::Series] {
|
||||
let selection_bit = selection_bit(cluster);
|
||||
if requested_selection_mask & selection_bit == 0 {
|
||||
continue;
|
||||
}
|
||||
let report_cluster = quality_cluster(cluster);
|
||||
let threshold = update_quality.threshold(report_cluster);
|
||||
if quality_bypass_mask & selection_bit != 0 {
|
||||
let candidate_count = count_candidate_cluster(app_config, storage_path, generation, cluster).await?;
|
||||
force_updates.push(ClusterForceUpdate {
|
||||
cluster: report_cluster,
|
||||
candidate_count,
|
||||
configured_threshold: threshold,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if threshold == 0 {
|
||||
continue;
|
||||
}
|
||||
let candidate_count = count_candidate_cluster(app_config, storage_path, generation, cluster).await?;
|
||||
let Some(active_manifest) = active_manifest else {
|
||||
return Err(TuliproxError::RepositoryStalker(format!(
|
||||
"Missing active manifest for enforced {report_cluster} quality evaluation"
|
||||
)));
|
||||
};
|
||||
let current_count = count_active_cluster(app_config, active_manifest, cluster).await?;
|
||||
let decision = evaluate_update_quality(current_count, candidate_count, threshold);
|
||||
if let Some(acceptance) = decision.acceptance(report_cluster) {
|
||||
quality_acceptances.push(acceptance);
|
||||
} else if let Some(rejection) = decision.rejection(report_cluster) {
|
||||
accepted_selection_mask &= !selection_bit;
|
||||
if cluster == StalkerCluster::Live {
|
||||
accepted_selection_mask &= !EPG_SELECTION;
|
||||
}
|
||||
quality_rejections.push(rejection);
|
||||
}
|
||||
}
|
||||
Ok(StalkerGenerationPublication { accepted_selection_mask, quality_acceptances, quality_rejections, force_updates })
|
||||
}
|
||||
|
||||
fn category_map(categories: Vec<StalkerCategory>) -> HashMap<u32, StalkerCategory> {
|
||||
@@ -317,12 +631,14 @@ pub async fn advance_stalker_refresh(
|
||||
app_config: &Arc<AppConfig>,
|
||||
api_client: &StalkerApiClient,
|
||||
handshake: &StalkerHandshake,
|
||||
selection: StalkerClusterSelection,
|
||||
refresh_plan: StalkerRefreshPlan,
|
||||
storage_path: &Path,
|
||||
identity_fingerprint: u64,
|
||||
mut budget: StalkerRefreshBudget,
|
||||
) -> Result<StalkerRefreshOutcome, TuliproxError> {
|
||||
let mut checkpoint = load_or_start_checkpoint(storage_path, identity_fingerprint, selection).await?;
|
||||
let StalkerRefreshPlan { selection, update_quality, quality_bypass_mask } = refresh_plan;
|
||||
let mut checkpoint =
|
||||
load_or_start_checkpoint(storage_path, identity_fingerprint, selection, quality_bypass_mask).await?;
|
||||
if checkpoint.phase == StalkerRefreshPhase::Terminal {
|
||||
return finish_terminal_refresh(storage_path).await;
|
||||
}
|
||||
@@ -568,8 +884,15 @@ pub async fn advance_stalker_refresh(
|
||||
checkpoint.retry_count = 0;
|
||||
}
|
||||
StalkerRefreshPhase::Complete => {
|
||||
finish_completed_refresh_checked(app_config, storage_path, identity_fingerprint, &checkpoint).await?;
|
||||
return Ok(StalkerRefreshOutcome::Complete);
|
||||
let reports = finish_completed_refresh_checked(
|
||||
app_config,
|
||||
storage_path,
|
||||
identity_fingerprint,
|
||||
&checkpoint,
|
||||
update_quality,
|
||||
)
|
||||
.await?;
|
||||
return Ok(reports.into_outcome());
|
||||
}
|
||||
StalkerRefreshPhase::Terminal => {
|
||||
return finish_terminal_refresh(storage_path).await;
|
||||
@@ -584,11 +907,17 @@ pub async fn advance_stalker_refresh(
|
||||
mod tests {
|
||||
use super::*;
|
||||
use arc_swap::{ArcSwap, ArcSwapOption};
|
||||
use shared::model::ConfigPaths;
|
||||
use shared::{
|
||||
model::{ConfigInputUpdateQualityDto, ConfigPaths},
|
||||
utils::Internable,
|
||||
};
|
||||
use tuliprox_core::{
|
||||
model::{ApiProxyConfig, Config, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig},
|
||||
utils::FileLockManager,
|
||||
};
|
||||
use tuliprox_repository::stalker_generation_repository::{
|
||||
load_active_manifest, save_active_manifest, ClusterFiles, SeriesFiles, StalkerActiveManifest,
|
||||
};
|
||||
|
||||
fn test_app_config(storage_dir: &Path) -> Arc<AppConfig> {
|
||||
Arc::new(AppConfig {
|
||||
@@ -620,6 +949,78 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
fn update_quality(live: u8, vod: u8, series: u8) -> ConfigInputUpdateQuality {
|
||||
ConfigInputUpdateQuality::from(&ConfigInputUpdateQualityDto { live, vod, series })
|
||||
}
|
||||
|
||||
fn stalker_items(count: usize, kind: StalkerStreamKind, series_roots: bool) -> Vec<StalkerPlaylistItem> {
|
||||
(0..count)
|
||||
.map(|index| StalkerPlaylistItem {
|
||||
stream_id: u32::try_from(index + 1).unwrap_or(u32::MAX),
|
||||
name: format!("item-{index}").intern(),
|
||||
stream_kind: kind,
|
||||
is_series: series_roots,
|
||||
..StalkerPlaylistItem::default()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn write_generation_cluster(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_path: &Path,
|
||||
generation: u64,
|
||||
cluster: StalkerCluster,
|
||||
primary_count: usize,
|
||||
episode_count: usize,
|
||||
) -> Result<(), TuliproxError> {
|
||||
let (primary_data, kind, series_roots) = match cluster {
|
||||
StalkerCluster::Live => (StalkerGenerationData::Live, StalkerStreamKind::Live, false),
|
||||
StalkerCluster::Vod => (StalkerGenerationData::Vod, StalkerStreamKind::Movie, false),
|
||||
StalkerCluster::Series => (StalkerGenerationData::SeriesRoots, StalkerStreamKind::Episode, true),
|
||||
};
|
||||
let primary_path = generation_data_path(storage_path, generation, primary_data);
|
||||
let primary_items = stalker_items(primary_count, kind, series_roots);
|
||||
if primary_items.is_empty() {
|
||||
upsert_stalker_items_at(app_config, &primary_path, &primary_items).await?;
|
||||
} else {
|
||||
snapshot_stalker_items_at(app_config, primary_path, &primary_items).await?;
|
||||
}
|
||||
if cluster == StalkerCluster::Series {
|
||||
let episode_path = generation_data_path(storage_path, generation, StalkerGenerationData::SeriesEpisodes);
|
||||
snapshot_stalker_items_at(
|
||||
app_config,
|
||||
episode_path,
|
||||
&stalker_items(episode_count, StalkerStreamKind::Episode, false),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn complete_manifest(storage_path: &Path, identity_fingerprint: u64, generation: u64) -> StalkerActiveManifest {
|
||||
StalkerActiveManifest {
|
||||
schema: StalkerActiveManifest::empty(identity_fingerprint).schema,
|
||||
identity_fingerprint,
|
||||
live: Some(ClusterFiles {
|
||||
generation,
|
||||
data: generation_data_path(storage_path, generation, StalkerGenerationData::Live),
|
||||
}),
|
||||
vod: Some(ClusterFiles {
|
||||
generation,
|
||||
data: generation_data_path(storage_path, generation, StalkerGenerationData::Vod),
|
||||
}),
|
||||
series: Some(SeriesFiles {
|
||||
generation,
|
||||
roots: generation_data_path(storage_path, generation, StalkerGenerationData::SeriesRoots),
|
||||
episodes: generation_data_path(storage_path, generation, StalkerGenerationData::SeriesEpisodes),
|
||||
}),
|
||||
epg: Some(ClusterFiles {
|
||||
generation,
|
||||
data: generation_data_path(storage_path, generation, StalkerGenerationData::Epg),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_checkpoint_is_cleared_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
@@ -643,13 +1044,33 @@ mod tests {
|
||||
checkpoint.phase = StalkerRefreshPhase::Complete;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
let resumed = load_or_start_checkpoint(temp.path(), 17, selection).await?;
|
||||
let resumed = load_or_start_checkpoint(temp.path(), 17, selection, 0).await?;
|
||||
|
||||
assert_eq!(resumed.generation, 23);
|
||||
assert_eq!(resumed.phase, StalkerRefreshPhase::Complete);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumed_checkpoint_preserves_and_extends_the_quality_bypass_mask() -> Result<(), Box<dyn std::error::Error>>
|
||||
{
|
||||
let temp = tempfile::tempdir()?;
|
||||
let selection = StalkerClusterSelection { live: true, vod: true, series: true, epg: false };
|
||||
let mut checkpoint = StalkerCheckpoint::new(17, 23, selection.mask(), 123);
|
||||
checkpoint.phase = StalkerRefreshPhase::Vod { page: 4 };
|
||||
checkpoint.quality_bypass_mask = LIVE_SELECTION;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
let resumed = load_or_start_checkpoint(temp.path(), 17, selection, SERIES_SELECTION).await?;
|
||||
|
||||
assert_eq!(resumed.generation, 23);
|
||||
assert_eq!(resumed.phase, StalkerRefreshPhase::Vod { page: 4 });
|
||||
assert_eq!(resumed.quality_bypass_mask, LIVE_SELECTION | SERIES_SELECTION);
|
||||
let resumed_without_new_force = load_or_start_checkpoint(temp.path(), 17, selection, 0).await?;
|
||||
assert_eq!(resumed_without_new_force.quality_bypass_mask, LIVE_SELECTION | SERIES_SELECTION);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_complete_refresh_keeps_the_active_manifest() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
@@ -666,9 +1087,16 @@ mod tests {
|
||||
checkpoint.phase = StalkerRefreshPhase::Complete;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
let result = finish_completed_refresh_checked(&app_config, temp.path(), 17, &checkpoint).await;
|
||||
let reports = finish_completed_refresh_checked(
|
||||
&app_config,
|
||||
temp.path(),
|
||||
17,
|
||||
&checkpoint,
|
||||
ConfigInputUpdateQuality::default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(reports.error.is_some());
|
||||
assert_eq!(load_active_manifest(temp.path(), 17).await?, active);
|
||||
assert!(load_checkpoint(temp.path(), 17).await?.is_none());
|
||||
Ok(())
|
||||
@@ -677,22 +1105,368 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn complete_publication_is_idempotent_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let mut checkpoint = StalkerCheckpoint::new(17, 23, 0b0011, 123);
|
||||
checkpoint.phase = StalkerRefreshPhase::Complete;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
finish_completed_refresh(temp.path(), 17, &checkpoint).await?;
|
||||
finish_completed_refresh(&app_config, temp.path(), 17, &checkpoint, ConfigInputUpdateQuality::default())
|
||||
.await?;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
finish_completed_refresh(temp.path(), 17, &checkpoint).await?;
|
||||
finish_completed_refresh(&app_config, temp.path(), 17, &checkpoint, ConfigInputUpdateQuality::default())
|
||||
.await?;
|
||||
|
||||
let manifest =
|
||||
tuliprox_repository::stalker_generation_repository::load_active_manifest(temp.path(), 17).await?;
|
||||
let manifest = load_active_manifest(temp.path(), 17).await?;
|
||||
assert_eq!(manifest.live.as_ref().map(|files| files.generation), Some(23));
|
||||
assert_eq!(manifest.vod.as_ref().map(|files| files.generation), Some(23));
|
||||
assert!(load_checkpoint(temp.path(), 17).await?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipeline_transparency_stalker_preserves_accepted_quality_facts_after_publication(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 17;
|
||||
let active_generation = 41;
|
||||
let candidate_generation = 42;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Vod, 100, 0).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Series, 40, 60).await?;
|
||||
let active = complete_manifest(temp.path(), identity, active_generation);
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 90, 0).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Vod, 89, 0).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Series, 45, 65)
|
||||
.await?;
|
||||
let checkpoint = StalkerCheckpoint::new(identity, candidate_generation, 0b1111, 123);
|
||||
|
||||
let reports = finish_completed_refresh_checked(
|
||||
&app_config,
|
||||
temp.path(),
|
||||
identity,
|
||||
&checkpoint,
|
||||
update_quality(90, 90, 90),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert!(reports.error.is_none());
|
||||
assert_eq!(
|
||||
reports.quality_acceptances,
|
||||
vec![
|
||||
ClusterUpdateAcceptance {
|
||||
cluster: XtreamCluster::Live,
|
||||
current_count: Some(100),
|
||||
candidate_count: 90,
|
||||
threshold: 90,
|
||||
quality: Some(90),
|
||||
},
|
||||
ClusterUpdateAcceptance {
|
||||
cluster: XtreamCluster::Series,
|
||||
current_count: Some(100),
|
||||
candidate_count: 110,
|
||||
threshold: 90,
|
||||
quality: Some(90),
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
reports.quality_rejections,
|
||||
vec![ClusterUpdateRejection {
|
||||
cluster: XtreamCluster::Video,
|
||||
current_count: 100,
|
||||
candidate_count: 89,
|
||||
threshold: 90,
|
||||
quality: 89,
|
||||
}]
|
||||
);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
assert_eq!(published.vod, active.vod);
|
||||
assert_eq!(published.series.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
assert_eq!(published.epg.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipeline_transparency_stalker_keeps_acceptance_when_pre_publish_validation_fails(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 18;
|
||||
let active_generation = 51;
|
||||
let candidate_generation = 52;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
|
||||
let mut active = StalkerActiveManifest::empty(identity);
|
||||
active.live = Some(ClusterFiles {
|
||||
generation: active_generation,
|
||||
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
|
||||
});
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 90, 0).await?;
|
||||
let mut checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION, 123);
|
||||
checkpoint.phase = StalkerRefreshPhase::Complete;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
let reports =
|
||||
finish_completed_refresh_checked(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0))
|
||||
.await?;
|
||||
|
||||
let StalkerRefreshOutcome::Failed { quality_acceptances, error: _, .. } = reports.into_outcome() else {
|
||||
return Err("expected failed Stalker completion with retained reports".into());
|
||||
};
|
||||
assert_eq!(
|
||||
quality_acceptances,
|
||||
vec![ClusterUpdateAcceptance {
|
||||
cluster: XtreamCluster::Live,
|
||||
current_count: Some(100),
|
||||
candidate_count: 90,
|
||||
threshold: 90,
|
||||
quality: Some(90),
|
||||
}]
|
||||
);
|
||||
assert_eq!(load_active_manifest(temp.path(), identity).await?, active);
|
||||
assert!(load_checkpoint(temp.path(), identity).await?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipeline_transparency_stalker_keeps_acceptance_when_cleanup_fails_after_publish(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 19;
|
||||
let active_generation = 61;
|
||||
let candidate_generation = 62;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
|
||||
let mut active = StalkerActiveManifest::empty(identity);
|
||||
active.live = Some(ClusterFiles {
|
||||
generation: active_generation,
|
||||
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
|
||||
});
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 90, 0).await?;
|
||||
tokio::fs::create_dir(generation_data_path(temp.path(), 1, StalkerGenerationData::Live)).await?;
|
||||
tokio::fs::write(generation_data_path(temp.path(), 2, StalkerGenerationData::Live), b"obsolete").await?;
|
||||
let mut checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION, 123);
|
||||
checkpoint.phase = StalkerRefreshPhase::Complete;
|
||||
checkpoint.processed = 90;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
let reports =
|
||||
finish_completed_refresh_checked(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0))
|
||||
.await?;
|
||||
|
||||
let StalkerRefreshOutcome::Failed { quality_acceptances, error: _, .. } = reports.into_outcome() else {
|
||||
return Err("expected failed Stalker completion with retained reports".into());
|
||||
};
|
||||
assert_eq!(quality_acceptances.len(), 1);
|
||||
assert_eq!(quality_acceptances[0].cluster, XtreamCluster::Live);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
assert!(load_checkpoint(temp.path(), identity).await?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_live_generation_keeps_active_live_and_epg() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 23;
|
||||
let active_generation = 7;
|
||||
let candidate_generation = 8;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 100, 0).await?;
|
||||
let mut active = StalkerActiveManifest::empty(identity);
|
||||
active.live = Some(ClusterFiles {
|
||||
generation: active_generation,
|
||||
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
|
||||
});
|
||||
active.epg = Some(ClusterFiles {
|
||||
generation: active_generation,
|
||||
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Epg),
|
||||
});
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 89, 0).await?;
|
||||
let checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION | EPG_SELECTION, 123);
|
||||
|
||||
let rejections =
|
||||
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0)).await?;
|
||||
|
||||
assert_eq!(rejections.len(), 1);
|
||||
assert_eq!(rejections[0].cluster, XtreamCluster::Live);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.live, active.live);
|
||||
assert_eq!(published.epg, active.epg);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nonempty_bootstrap_generation_is_published() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 29;
|
||||
let generation = 3;
|
||||
write_generation_cluster(&app_config, temp.path(), generation, StalkerCluster::Series, 2, 3).await?;
|
||||
let checkpoint = StalkerCheckpoint::new(identity, generation, SERIES_SELECTION, 123);
|
||||
|
||||
let rejections =
|
||||
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(0, 0, 100))
|
||||
.await?;
|
||||
|
||||
assert!(rejections.is_empty());
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.series.as_ref().map(|files| files.generation), Some(generation));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_bootstrap_generation_is_rejected() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 30;
|
||||
let generation = 4;
|
||||
write_generation_cluster(&app_config, temp.path(), generation, StalkerCluster::Live, 0, 0).await?;
|
||||
let checkpoint = StalkerCheckpoint::new(identity, generation, LIVE_SELECTION | EPG_SELECTION, 123);
|
||||
|
||||
let rejections =
|
||||
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(90, 0, 0)).await?;
|
||||
|
||||
assert_eq!(
|
||||
rejections,
|
||||
vec![ClusterUpdateRejection {
|
||||
cluster: XtreamCluster::Live,
|
||||
current_count: 0,
|
||||
candidate_count: 0,
|
||||
threshold: 90,
|
||||
quality: 0,
|
||||
}]
|
||||
);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert!(published.live.is_none());
|
||||
assert!(published.epg.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_empty_stalker_generation_is_published_without_a_quality_rejection(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 40;
|
||||
let active_generation = 5;
|
||||
let candidate_generation = 6;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Vod, 10, 0).await?;
|
||||
let mut active = complete_manifest(temp.path(), identity, active_generation);
|
||||
active.live = None;
|
||||
active.series = None;
|
||||
active.epg = None;
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Vod, 0, 0).await?;
|
||||
let mut checkpoint = StalkerCheckpoint::new(identity, candidate_generation, VOD_SELECTION, 123);
|
||||
checkpoint.phase = StalkerRefreshPhase::Complete;
|
||||
checkpoint.quality_bypass_mask = VOD_SELECTION;
|
||||
save_checkpoint(temp.path(), &checkpoint).await?;
|
||||
|
||||
let reports = finish_completed_refresh_checked(
|
||||
&app_config,
|
||||
temp.path(),
|
||||
identity,
|
||||
&checkpoint,
|
||||
update_quality(0, 100, 0),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert!(reports.quality_rejections.is_empty());
|
||||
assert_eq!(
|
||||
reports.force_updates,
|
||||
vec![ClusterForceUpdate { cluster: XtreamCluster::Video, candidate_count: 0, configured_threshold: 100 }]
|
||||
);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.vod.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
assert!(load_checkpoint(temp.path(), identity).await?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_live_generation_publishes_its_epg_with_the_same_generation(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 41;
|
||||
let active_generation = 7;
|
||||
let candidate_generation = 8;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 10, 0).await?;
|
||||
let mut active = StalkerActiveManifest::empty(identity);
|
||||
active.live = Some(ClusterFiles {
|
||||
generation: active_generation,
|
||||
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Live),
|
||||
});
|
||||
active.epg = Some(ClusterFiles {
|
||||
generation: active_generation,
|
||||
data: generation_data_path(temp.path(), active_generation, StalkerGenerationData::Epg),
|
||||
});
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 1, 0).await?;
|
||||
let mut checkpoint =
|
||||
StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION | EPG_SELECTION, 123);
|
||||
checkpoint.quality_bypass_mask = LIVE_SELECTION;
|
||||
|
||||
let reports = finish_completed_refresh_checked(
|
||||
&app_config,
|
||||
temp.path(),
|
||||
identity,
|
||||
&checkpoint,
|
||||
update_quality(100, 0, 0),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert!(reports.quality_rejections.is_empty());
|
||||
assert_eq!(reports.force_updates.len(), 1);
|
||||
assert_eq!(reports.force_updates[0].cluster, XtreamCluster::Live);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
assert_eq!(published.epg.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_series_counts_roots_and_episodes_without_blocking_accepted_live_generation(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let app_config = test_app_config(temp.path());
|
||||
let identity = 31;
|
||||
let active_generation = 11;
|
||||
let candidate_generation = 12;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Live, 10, 0).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), active_generation, StalkerCluster::Series, 2, 8).await?;
|
||||
let mut active = complete_manifest(temp.path(), identity, active_generation);
|
||||
active.vod = None;
|
||||
save_active_manifest(temp.path(), &active).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Live, 10, 0).await?;
|
||||
write_generation_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Series, 2, 7).await?;
|
||||
let checkpoint = StalkerCheckpoint::new(identity, candidate_generation, LIVE_SELECTION | SERIES_SELECTION, 123);
|
||||
|
||||
assert_eq!(count_active_cluster(&app_config, &active, StalkerCluster::Series).await?, Some(10));
|
||||
assert_eq!(
|
||||
count_candidate_cluster(&app_config, temp.path(), candidate_generation, StalkerCluster::Series).await?,
|
||||
9
|
||||
);
|
||||
|
||||
let rejections =
|
||||
finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(100, 0, 100))
|
||||
.await?;
|
||||
|
||||
assert_eq!(rejections.len(), 1);
|
||||
assert_eq!(rejections[0].cluster, XtreamCluster::Series);
|
||||
assert_eq!(rejections[0].current_count, 10);
|
||||
assert_eq!(rejections[0].candidate_count, 9);
|
||||
let published = load_active_manifest(temp.path(), identity).await?;
|
||||
assert_eq!(published.live.as_ref().map(|files| files.generation), Some(candidate_generation));
|
||||
assert_eq!(published.series, active.series);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unit_budget_yields_after_completed_unit() {
|
||||
let mut budget = StalkerRefreshBudget::units(1);
|
||||
|
||||
Reference in New Issue
Block a user