Feature/staged input and path improvements (#632)

* No template file not found error message
* Setup wizard improvmenets
* path improvements
* staged input fix
* path improvements
* Provider dns resolved ips now writing to a file instead of source.yml
* Playlist update hard kill after 1h
* matadata update tmdb fix
* Web UI fixes
* GeoIp update schedule
* batch:// prefix  for aliases

backend/src/processing/processor/playlist.rs

@@ -50,13 +50,15 @@ use std::{
     collections::{HashMap, HashSet},
     path::PathBuf,
     sync::{Arc, Weak},
-    time::Instant,
+    time::{Duration, Instant},
 };
 use tokio::{
     sync::{Mutex, OwnedRwLockWriteGuard, RwLock},
     task::JoinSet,
 };
 
+const PLAYLIST_UPDATE_MAX_DURATION_SECS: u64 = 3600;
+
 fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool {
     let provider = ValueProvider { pli, match_as_ascii };
     filter.filter(&provider)
@@ -324,10 +326,11 @@ async fn playlist_download_from_input(
     input: &ConfigInput,
 ) -> PlaylistDownloadResult {
     let config = &*app_config.config.load();
-    let working_dir = &config.working_dir;
+    let storage_dir = &config.storage_dir;
+    let download_input_type = input.get_download_input_type();
 
     // Check Status
-    let storage_path = input_cache::resolve_input_storage_path(working_dir, &input.name).await;
+    let storage_path = input_cache::resolve_input_storage_path(storage_dir, &input.name).await;
     let mut status = input_cache::load_input_status(&storage_path);
     let cache_duration = input.cache_duration_seconds;
 
@@ -336,7 +339,7 @@ async fn playlist_download_from_input(
         let _ = std::fs::create_dir_all(&storage_path);
     }
 
-    let (clusters_to_download, fully_cached) = match input.input_type {
+    let (clusters_to_download, fully_cached) = match download_input_type {
         InputType::Xtream => {
             let mut to_download = vec![];
             for c in XTREAM_CLUSTER {
@@ -364,7 +367,7 @@ async fn playlist_download_from_input(
         return PlaylistDownloadResult::new(vec![], vec![], true, false);
     }
 
-    let (playlist, errors, persisted) = match input.input_type {
+    let (playlist, errors, persisted) = match download_input_type {
         InputType::M3u => {
             let (p, e) = m3u::download_m3u_playlist(app_config, client, config, input).await;
             (p, e, false)
@@ -381,7 +384,7 @@ async fn playlist_download_from_input(
 
     // Update Status
     if errors.is_empty() {
-        if let InputType::Xtream = input.input_type {
+        if let InputType::Xtream = download_input_type {
             if let Some(clusters) = clusters_to_download {
                 for c in clusters {
                     input_cache::update_cluster_status(&mut status, &c.to_string(), ClusterState::Ok);
@@ -401,7 +404,7 @@ async fn playlist_download_from_input(
         // We could mark specific clusters as failed if we knew which one failed.
         // For simplicity, if error, we don't update the timestamp (so it stays expired/invalid).
         // Or we mark as Failed.
-        if let InputType::Xtream = input.input_type {
+        if let InputType::Xtream = download_input_type {
             if let Some(clusters) = clusters_to_download {
                 for c in clusters {
                     // Optimistic: Only mark failed if we are sure?
@@ -437,6 +440,7 @@ async fn process_source(
                 continue;
             };
             if is_input_enabled(input, &ctx.user_targets) {
+                let effective_input_type = input.get_download_input_type();
                 source_downloaded = true;
                 log_memory_snapshot(format!("source[{source_idx}] input '{}' before_download", input.name).as_str());
 
@@ -459,7 +463,7 @@ async fn process_source(
                 };
                 log_memory_snapshot(format!("source[{source_idx}] input '{}' after_download", input.name).as_str());
 
-                let tvguide = if input.input_type == InputType::Library {
+                let tvguide = if effective_input_type == InputType::Library {
                     None
                 } else {
                     download_input_epg(ctx, input, &mut error_list).await
@@ -483,7 +487,7 @@ async fn process_source(
                 let elapsed = start_time.elapsed().as_secs();
                 input_stats.insert(
                     input_name.clone(),
-                    create_input_stat(group_count, channel_count, errors.len(), input.input_type, input_name, elapsed),
+                    create_input_stat(group_count, channel_count, errors.len(), effective_input_type, input_name, elapsed),
                 );
             }
         }
@@ -547,8 +551,8 @@ async fn download_input_epg(
 ) -> Option<TVGuide> {
     // Download epg for input
     let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
-        let working_dir = &ctx.config.config.load().working_dir;
-        epg::get_xmltv(ctx, input, None, working_dir).await
+        let storage_dir = &ctx.config.config.load().storage_dir;
+        epg::get_xmltv(ctx, input, None, storage_dir).await
     } else {
         (None, vec![])
     };
@@ -1018,10 +1022,10 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
     }
 
     let input_name = fpl.input.name.clone();
-    let input_type = fpl.input.input_type;
-    let xtream_probe_handled = input_type.is_xtream() && target.get_xtream_output().is_some();
+    let effective_input_type = fpl.input.get_download_input_type();
+    let xtream_probe_handled = effective_input_type.is_xtream() && target.get_xtream_output().is_some();
     let live_probe_settings = if probe_live_enabled {
-        get_live_probe_interval_settings(target, input_type, Some(opts)).map(|(delay, interval_secs)| {
+        get_live_probe_interval_settings(target, effective_input_type, Some(opts)).map(|(delay, interval_secs)| {
             let interval_signed = i64::try_from(interval_secs).unwrap_or(i64::MAX);
             let cutoff_ts = chrono::Utc::now().timestamp().saturating_sub(interval_signed);
             (delay, interval_secs, cutoff_ts)
@@ -1050,6 +1054,16 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
                     if needs_live_probe(&item, cutoff_ts) {
                         if let Some(provider_id) = provider_id_from_item(&item) {
                             if queued_live_keys.insert(provider_id.clone()) {
+                                if log_enabled!(Level::Debug) {
+                                    let last_probed = match item.header.additional_properties.as_ref() {
+                                        Some(StreamProperties::Live(props)) => props.last_probed_timestamp,
+                                        _ => None,
+                                    };
+                                    debug!(
+                                        "[Task] Creating ProbeLive task for input {}: id={}, last_probed_ts={:?}, cutoff_ts={}, interval={}s, title=\"{}\"",
+                                        input_name, provider_id, last_probed, cutoff_ts, interval_secs, item.header.title
+                                    );
+                                }
                                 let task = UpdateTask::ProbeLive {
                                     id: provider_id,
                                     reason: ResolveReasonSet::from_variants(&[ResolveReason::Probe]),
@@ -1092,7 +1106,7 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
         }
 
         // For M3U, ID is a provider id; for Library, ID is UUID.
-        let unique_id = if input_type == InputType::Library {
+        let unique_id = if effective_input_type == InputType::Library {
             item.header.uuid.to_valid_uuid()
         } else {
             item.header.id.to_string()
@@ -1104,6 +1118,10 @@ async fn playlist_probe(ctx: &PlaylistProcessingContext, target: &ConfigTarget,
             continue;
         }
 
+        debug!(
+            "[Task] Creating ProbeStream task for input {}: scope={}, unique_id={}, item_type={:?}, title=\"{}\"",
+            input_name, probe_scope, unique_id, item.header.item_type, item.header.title
+        );
         let task = UpdateTask::ProbeStream {
             probe_scope,
             unique_id,
@@ -1204,7 +1222,7 @@ async fn process_watch(
     }
 }
 
-#[allow(clippy::too_many_arguments)]
+#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
 pub async fn exec_processing(
     client: &reqwest::Client,
     app_config: Arc<AppConfig>,
@@ -1219,6 +1237,7 @@ pub async fn exec_processing(
     pre_processed_inputs: Option<HashSet<Arc<str>>>,
     acquired_permit: Option<crate::api::model::UpdateGuardPermit>,
 ) {
+    let max_update_duration = Duration::from_secs(PLAYLIST_UPDATE_MAX_DURATION_SECS);
     let playlist_guard = if let Some(permit) = acquired_permit {
         Some(permit)
     } else if let Some(guard) = &update_guard {
@@ -1237,7 +1256,15 @@ pub async fn exec_processing(
 
     if playlist_guard.is_some() {
         if let Some(state) = app_state.as_ref() {
-            sync_panel_api_exp_dates(state).await;
+            if tokio::time::timeout(max_update_duration, sync_panel_api_exp_dates(state)).await.is_err() {
+                error!(
+                    "Playlist update bootstrap timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
+                );
+                if let Some(events) = event_manager.as_deref() {
+                    events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
+                }
+                return;
+            }
         }
     }
 
@@ -1266,13 +1293,29 @@ pub async fn exec_processing(
     };
 
     let start_time = Instant::now();
-    let process_result = std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind().await;
-    let Ok((stats, errors)) = process_result else {
-        error!("Playlist processing panicked");
-        if let Some(events) = event_manager.as_deref() {
-            events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
+    let process_result = tokio::time::timeout(
+        max_update_duration,
+        std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind(),
+    )
+    .await;
+    let (stats, errors) = match process_result {
+        Ok(Ok((stats, errors))) => (stats, errors),
+        Ok(Err(_)) => {
+            error!("Playlist processing panicked");
+            if let Some(events) = event_manager.as_deref() {
+                events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
+            }
+            return;
+        }
+        Err(_) => {
+            error!(
+                "Playlist processing timed out after {PLAYLIST_UPDATE_MAX_DURATION_SECS} secs while holding playlist lock",
+            );
+            if let Some(events) = event_manager.as_deref() {
+                events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure));
+            }
+            return;
         }
-        return;
     };
     log_memory_snapshot("exec_processing after_process_sources");
 

backend/src/processing/processor/stream_probe.rs

@@ -41,7 +41,7 @@ pub async fn update_generic_stream_metadata(
     active_provider: &Arc<ActiveProviderManager>,
     active_handle: Option<&crate::api::model::ProviderHandle>,
 ) -> Result<GenericProbeOutcome, TuliproxError> {
-    let working_dir = &app_config.config.load().working_dir;
+    let storage_dir = &app_config.config.load().storage_dir;
 
     // Check if probing is enabled globally
     let ffprobe_enabled = app_config.is_ffprobe_enabled().await;
@@ -50,7 +50,7 @@ pub async fn update_generic_stream_metadata(
     }
 
     // Determine storage file path based on input type
-    let storage_path = get_input_storage_path(&input.name, working_dir).await
+    let storage_path = get_input_storage_path(&input.name, storage_dir).await
         .map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
 
     let (db_path, storage_kind) = match input.input_type {

backend/src/processing/processor/xtream.rs

@@ -21,8 +21,8 @@ pub async fn update_live_stream_metadata(
     _active_handle: Option<&ProviderHandle>,
     _active_provider: &Arc<ActiveProviderManager>,
 ) -> Result<Option<LiveStreamProperties>, TuliproxError> {
-    let working_dir = &app_config.config.load().working_dir;
-    let storage_path = get_input_storage_path(&input.name, working_dir).await
+    let storage_dir = &app_config.config.load().storage_dir;
+    let storage_path = get_input_storage_path(&input.name, storage_dir).await
         .map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
 
     // Try to load existing info first to preserve data

backend/src/processing/processor/xtream_series.rs

@@ -207,6 +207,19 @@ fn queue_background_series_info(
             let reasons = check_resolve_reasons(resolve_options, resolve_tmdb_enabled, pli);
 
             if !reasons.is_empty() {
+                if log_enabled!(Level::Debug) {
+                    let has_details = pli.has_details();
+                    let (has_tmdb, has_date) = match pli.header.additional_properties.as_ref() {
+                        Some(StreamProperties::Series(props)) => {
+                            (props.tmdb.is_some(), props.release_date.is_some())
+                        }
+                        _ => (false, false),
+                    };
+                    debug!(
+                        "[Task] Creating ResolveSeries task for input {}: id={}, reasons={}, has_details={}, has_tmdb={}, has_date={}, title=\"{}\"",
+                        input_name_arc, provider_id, reasons, has_details, has_tmdb, has_date, pli.header.title
+                    );
+                }
                 let task = UpdateTask::ResolveSeries {
                     id: provider_id,
                     reason: reasons,
@@ -233,8 +246,8 @@ async fn process_immediate_series_info(
     resolve_tmdb_enabled: bool,
 ) -> Vec<PlaylistGroup> {
     let input = fpl.input;
-    let working_dir = &ctx.config.config.load().working_dir;
-    let storage_path = match get_input_storage_path(&input.name, working_dir).await {
+    let storage_dir = &ctx.config.config.load().storage_dir;
+    let storage_path = match get_input_storage_path(&input.name, storage_dir).await {
         Ok(path) => path,
         Err(err) => {
             error!("Can't resolve series, input storage directory for input '{}' failed: {err}", input.name);
@@ -624,8 +637,8 @@ pub async fn update_series_metadata(
     db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
     tmdb_and_date_present_out: Option<&AtomicBool>,
 ) -> Result<Option<SeriesStreamProperties>, TuliproxError> {
-    let working_dir = &app_config.config.load().working_dir;
-    let storage_path = get_input_storage_path(&input.name, working_dir)
+    let storage_dir = &app_config.config.load().storage_dir;
+    let storage_path = get_input_storage_path(&input.name, storage_dir)
         .await
         .map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
 

backend/src/processing/processor/xtream_vod.rs

@@ -5,6 +5,7 @@ use crate::model::FetchedPlaylist;
 use crate::model::InputSource;
 use crate::model::{AppConfig, ConfigTarget};
 use crate::model::{ConfigInput, ConfigInputFlags};
+use crate::processing::input_cache::resolve_input_storage_path;
 use crate::processing::processor::playlist::PlaylistProcessingContext;
 use crate::processing::processor::{
     create_resolve_options_function_for_xtream_target, process_foreground_retry_once, select_cancel_token,
@@ -12,7 +13,6 @@ use crate::processing::processor::{
     FOREGROUND_RETRY_BATCH_MAX_SIZE as RETRY_BATCH_MAX_SIZE,
 };
 use crate::ptt::ptt_parse_title;
-use crate::repository::get_input_storage_path;
 use crate::repository::persist_input_vod_info;
 use crate::repository::persist_input_vod_info_batch;
 use crate::repository::{xtream_get_file_path, BPlusTreeQuery};
@@ -148,15 +148,9 @@ async fn process_immediate_vod_info(
     resolve_tmdb_enabled: bool,
 ) {
     let input = fpl.input;
-    let working_dir = &ctx.config.config.load().working_dir;
+    let storage_dir = &ctx.config.config.load().storage_dir;
 
-    let storage_path = match get_input_storage_path(&input.name, working_dir).await {
-        Ok(path) => path,
-        Err(err) => {
-            error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name);
-            return;
-        }
-    };
+    let storage_path = resolve_input_storage_path(storage_dir, &input.name).await;
 
     // Keep an optional read query open and reopen lazily only when needed.
     let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Video);
@@ -447,6 +441,26 @@ fn queue_background_vod_info(
         let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli);
 
         if !reasons.is_empty() {
+            if log_enabled!(Level::Debug) {
+                let has_details = pli.has_details();
+                let (has_tmdb, has_date, has_video, has_audio) =
+                    match pli.header.additional_properties.as_ref() {
+                        Some(StreamProperties::Video(props)) => {
+                            let details = props.details.as_ref();
+                            (
+                                props.tmdb.is_some(),
+                                details.and_then(|d| d.release_date.as_ref()).is_some(),
+                                MediaQuality::is_valid_media_info(details.and_then(|d| d.video.as_deref())),
+                                MediaQuality::is_valid_media_info(details.and_then(|d| d.audio.as_deref())),
+                            )
+                        }
+                        _ => (false, false, false, false),
+                    };
+                debug!(
+                    "[Task] Creating ResolveVod task for input {}: id={}, reasons={}, has_details={}, has_tmdb={}, has_date={}, has_video_info={}, has_audio_info={}, title=\"{}\"",
+                    input.name, provider_id, reasons, has_details, has_tmdb, has_date, has_video, has_audio, pli.header.title
+                );
+            }
             let task =
                 UpdateTask::ResolveVod { id: provider_id, reason: reasons, delay: resolve_options.resolve_delay };
             mgr.queue_task_background(input.name.clone(), task);
@@ -509,10 +523,8 @@ pub async fn update_vod_metadata(
     db_query: Option<Arc<Mutex<BPlusTreeQuery<u32, XtreamPlaylistItem>>>>,
     tmdb_resolved_out: Option<&AtomicBool>,
 ) -> Result<Option<VideoStreamProperties>, TuliproxError> {
-    let working_dir = &app_config.config.load().working_dir;
-    let storage_path = get_input_storage_path(&input.name, working_dir)
-        .await
-        .map_err(|e| shared::error::info_err!("Storage path error: {e}"))?;
+    let storage_dir = &app_config.config.load().storage_dir;
+    let storage_path = resolve_input_storage_path(storage_dir, &input.name).await;
 
     // Check if we should skip based on input options
     if input.has_flag(ConfigInputFlags::XtreamSkipVod) {