diff --git a/CHANGELOG.md b/CHANGELOG.md index c0445b67a..1ab8fd0cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ - `threads` attribute in `config.yml` renamed to `process_parallel` (boolean). - Added mandatory `rewrite_secret` to `reverse_proxy` config for stable resource URLs. - Removed `forced_retry_interval_secs`. + - FFprobe settings moved from `video.*` to `metadata_update.ffprobe.*`. + - `metadata_update.ffprobe.analyze_duration` and `metadata_update.ffprobe.live_analyze_duration` now require explicit unit suffixes (`s|m|h|d`). - **Input Batch Changes**: `name` attribute is now mandatory for input type batch to ensure stable playlist UUIDs. - **Favorites Redesign**: Replaced implicit `create_alias` with explicit `add_favourite(group_name)` script function. - **EpgSmartMatch**: Field `name_prefix` syntax needs to be changed from `name_prefix: !suffix "."` to `name_prefix: { suffix: "." }`. @@ -105,14 +107,19 @@ active URL of the specified provider. - Added `-T, --template` to override `template_path` on startup. - **Metadata Update Runtime Config**: Metadata worker intervals, retry/backoff limits, queue sizing, and probe cooldowns are now configurable through a dedicated `metadata_update` config block. +- **Unified Metadata Retry State**: Replaced probe-only retry persistence with a single `metadata_retry_state.db` per input. A single record per + item now stores retry/cooldown state for `resolve`, `probe`, and `tmdb`. +- **TMDB No-Match Cooldown**: Added explicit TMDB cooldown handling. When TMDB resolve completes successfully but returns no match, TMDB reasons are + suppressed for that item during cooldown to prevent endless requeue loops. ## ⚙️ New Settings - **config.yml**: - - Added `metadata_update` (optional) with queue/retry/backoff/cooldown/worker-timeout fields (duration syntax: `60`, `10m`, `1h`, `7d`). - - Added `metadata_update.ffprobe_enabled` (default: false), `metadata_update.ffprobe_timeout`, and ffprobe probe/analyze size settings. - - `metadata_update.ffprobe_analyze_duration` and `metadata_update.ffprobe_live_analyze_duration` now require explicit unit suffixes (`s|m|h|d`). - - FFprobe settings are now configured under `metadata_update` (not under `video` anymore). + - Added `metadata_update` (optional) with grouped sections: `log`, `resolve`, `probe`, `ffprobe`, `tmdb`. + - Added `metadata_update.tmdb.cooldown` (default `7d`) for successful TMDB no-match cooldown behavior. + - Added `metadata_update.ffprobe.enabled` (default: false), `metadata_update.ffprobe.timeout`, and ffprobe probe/analyze size settings. + - `metadata_update.ffprobe.analyze_duration` and `metadata_update.ffprobe.live_analyze_duration` require explicit unit suffixes (`s|m|h|d`). + - FFprobe settings are configured under `metadata_update.ffprobe` (not under `video`). - Added `template_path` (optional): path to a template file (`template.yml`) or directory (`template.d` style). - **source.yml (input options)**: - Added `resolve_tmdb`: Triggers TMDB lookup if ID is missing. diff --git a/README.md b/README.md index f924ee4a5..17d93e50b 100644 --- a/README.md +++ b/README.md @@ -385,67 +385,119 @@ Duration fields use the same format as `cache_duration`: plain seconds (`60`) or ```yaml metadata_update: - queue_log_interval: 30s - progress_log_interval: 15s - max_resolve_retry_backoff: 1h - resolve_min_retry_base: 5s - max_attempts_resolve: 3 - max_attempts_probe: 3 - resolve_exhaustion_reset_gap: 1h - probe_cooldown: 7d + log: + queue_interval: 30s + progress_interval: 15s + resolve: + max_retry_backoff: 1h + min_retry_base: 5s + max_attempts: 3 + exhaustion_reset_gap: 1h + probe: + cooldown: 7d + retry_load_retry_delay: 1m + retry_backoff_step_1: 10m + retry_backoff_step_2: 30m + retry_backoff_step_3: 1h + max_attempts: 3 + backoff_jitter_percent: 20 + tmdb: + enabled: true + # api_key: "..." # Optional, fallback is internal default placeholder + rate_limit_ms: 250 + cache_duration_days: 0 + language: en-US + cooldown: 7d + ffprobe: + enabled: true + timeout: 60 + analyze_duration: 10s + probe_size: 10MB + live_analyze_duration: 5s + live_probe_size: 5MB retry_delay: 2s - probe_retry_load_retry_delay: 1m worker_idle_timeout: 1m - probe_retry_backoff_step_1: 10m - probe_retry_backoff_step_2: 30m - probe_retry_backoff_step_3: 1h - backoff_jitter_percent: 20 max_queue_size: 100000 - ffprobe_enabled: true - ffprobe_timeout: 60 - ffprobe_analyze_duration: 10s - ffprobe_probe_size: 10MB - ffprobe_live_analyze_duration: 5s - ffprobe_live_probe_size: 5MB ``` **Field Reference (`metadata_update`):** -- `queue_log_interval` (default `30s`): Interval for queue size/status log output of the metadata worker. -- `progress_log_interval` (default `15s`): Interval for progress log output while metadata tasks are being processed. -- `max_resolve_retry_backoff` (default `1h`): Upper limit for resolve retry backoff delay. -- `resolve_min_retry_base` (default `5s`): Minimum base delay for resolve retries before exponential backoff is applied. -- `max_attempts_resolve` (default `3`): Maximum resolve attempts before a resolve task is treated as exhausted for the current cycle. -- `max_attempts_probe` (default `3`): Maximum probe attempts before probe cooldown is activated. -- `resolve_exhaustion_reset_gap` (default `1h`): Time gap after a completed update cycle after which exhausted resolve states are reset. -- `probe_cooldown` (default `7d`): Cooldown duration after probe attempts are exhausted; probe retries are skipped during this period. +**Group overview:** + +- `resolve`: Controls retries for metadata lookup tasks (for example title/date/TMDB resolution). +- `probe`: Controls retries and cooldown for technical stream probing tasks. +- `ffprobe`: Controls how deep and how long FFprobe inspects streams. +- `tmdb`: Controls TMDB API usage, language, caching, and no-match cooldown behavior. +- `log`: Controls how often metadata queue/progress messages are written to logs. + +- `log.queue_interval` (default `30s`): Interval for queue size/status log output of the metadata worker. +- `log.progress_interval` (default `15s`): Interval for progress log output while metadata tasks are being processed. +- `resolve.max_retry_backoff` (default `1h`): Upper limit for resolve retry backoff delay. +- `resolve.min_retry_base` (default `5s`): Minimum base delay for resolve retries before exponential backoff is applied. +- `resolve.max_attempts` (default `3`): Maximum resolve attempts before a resolve task is treated as exhausted for the current cycle. +- `resolve.exhaustion_reset_gap` (default `1h`): Time gap after a completed update cycle after which exhausted resolve states are reset. +- `probe.max_attempts` (default `3`): Maximum probe attempts before probe cooldown is activated. +- `probe.cooldown` (default `7d`): Cooldown duration after probe attempts are exhausted; probe retries are skipped during this period. +- `probe.retry_load_retry_delay` (default `1m`): Delay before re-attempting to load persisted metadata retry state after a load failure. +- `probe.retry_backoff_step_1` (default `10m`): Probe backoff delay for attempt 1. +- `probe.retry_backoff_step_2` (default `30m`): Probe backoff delay for attempt 2. +- `probe.retry_backoff_step_3` (default `1h`): Probe backoff delay for attempt 3 and higher. +- `probe.backoff_jitter_percent` (default `20`): Random jitter percentage applied to resolve/probe retry backoff to avoid synchronized retries. +- `tmdb.cooldown` (default `7d`): Cooldown duration after a TMDB lookup completed successfully but returned no match. +- `tmdb.enabled` / `tmdb.api_key` / `tmdb.rate_limit_ms` / `tmdb.cache_duration_days` / `tmdb.language`: TMDB resolver settings. - `retry_delay` (default `2s`): Minimum retry delay for transient worker errors (for example temporary connection/resource issues). -- `probe_retry_load_retry_delay` (default `1m`): Delay before re-attempting to load persisted probe retry state after a load failure. - `worker_idle_timeout` (default `1m`): Idle timeout for metadata worker shutdown when there is no immediate work. -- `probe_retry_backoff_step_1` (default `10m`): Probe backoff delay for attempt 1. -- `probe_retry_backoff_step_2` (default `30m`): Probe backoff delay for attempt 2. -- `probe_retry_backoff_step_3` (default `1h`): Probe backoff delay for attempt 3 and higher. -- `backoff_jitter_percent` (default `20`): Random jitter percentage applied to resolve/probe retry backoff to avoid synchronized retries. - `max_queue_size` (default `100000`): Maximum pending metadata tasks per input before new tasks are rejected. -- `ffprobe_enabled` (default `false`): Globally enables/disables FFprobe-based stream analysis. -- `ffprobe_timeout` (default `60`): FFprobe process timeout in seconds. -- `ffprobe_analyze_duration` (default `10s`): FFprobe `-analyzeduration` +- `ffprobe.enabled` (default `false`): Globally enables/disables FFprobe-based stream analysis. +- `ffprobe.timeout` (default `60`): FFprobe process timeout in seconds. +- `ffprobe.analyze_duration` (default `10s`): FFprobe `-analyzeduration` value for VOD/Series probing. Requires explicit unit suffix (`s`, `m`, `h`, `d`). -- `ffprobe_probe_size` (default `10MB`): FFprobe `-probesize` value for VOD/Series probing. -- `ffprobe_live_analyze_duration` (default `5s`): FFprobe `-analyzeduration` +- `ffprobe.probe_size` (default `10MB`): FFprobe `-probesize` value for VOD/Series probing. +- `ffprobe.live_analyze_duration` (default `5s`): FFprobe `-analyzeduration` value for Live probing. Requires explicit unit suffix (`s`, `m`, `h`, `d`). -- `ffprobe_live_probe_size` (default `5MB`): FFprobe `-probesize` value for Live probing. +- `ffprobe.live_probe_size` (default `5MB`): FFprobe `-probesize` value for Live probing. + +**Explanations:** + +- `retry_delay`: Minimum waiting time before a task is retried after a temporary problem. Prevents very fast retry loops. +- `worker_idle_timeout`: How long a metadata worker stays alive with no new work before it can stop and free resources. +- `max_queue_size`: Safety limit for how many metadata tasks are kept in memory at once. +- `log.queue_interval`: How often Tuliprox writes queue status logs. +- `log.progress_interval`: How often Tuliprox writes progress logs while tasks are running. +- `resolve.max_retry_backoff`: Longest wait time between repeated resolve retries. +- `resolve.min_retry_base`: Smallest wait time before a resolve retry is allowed. +- `resolve.max_attempts`: How many resolve tries are allowed for one item in one update cycle. +- `resolve.exhaustion_reset_gap`: How long Tuliprox waits before exhausted resolve items can start a fresh retry cycle. +- `probe.cooldown`: Pause period after probe retries are exhausted; item probing is skipped during that time. +- `probe.retry_load_retry_delay`: Wait time before retrying to read saved retry-state data after a load error. +- `probe.retry_backoff_step_1`: Wait time after the first probe failure. +- `probe.retry_backoff_step_2`: Wait time after the second probe failure. +- `probe.retry_backoff_step_3`: Wait time after the third and later probe failures. +- `probe.max_attempts`: How many probe tries are allowed before cooldown starts. +- `probe.backoff_jitter_percent`: Adds small random spread to retry timing so many tasks do not retry at the exact same second. +- `ffprobe.enabled`: Turns technical stream analysis on or off. +- `ffprobe.timeout`: Maximum runtime of one FFprobe call. Leave empty for default (`60`). +- `ffprobe.analyze_duration`: How much playback time FFprobe may inspect for VOD/Series. +- `ffprobe.probe_size`: How much data FFprobe may read for VOD/Series. +- `ffprobe.live_analyze_duration`: How much playback time FFprobe may inspect for live streams. +- `ffprobe.live_probe_size`: How much data FFprobe may read for live streams. +- `tmdb.enabled`: Enables TMDB metadata lookups. +- `tmdb.api_key`: Optional own TMDB API key. +- `tmdb.rate_limit_ms`: Minimum wait between TMDB requests (higher is gentler for the API). +- `tmdb.cache_duration_days`: How long TMDB results are cached before refresh. +- `tmdb.language`: Preferred language for TMDB metadata. +- `tmdb.cooldown`: Pause after a successful TMDB lookup with no match, to avoid endless retries. Duration fields support `s`, `m`, `h`, `d` (for example `30s`, `10m`, `1h`, `7d`) or plain seconds. -Exception: `ffprobe_analyze_duration` and `ffprobe_live_analyze_duration` require explicit unit suffix. +Exception: `ffprobe.analyze_duration` and `ffprobe.live_analyze_duration` require explicit unit suffix. Size fields support `B`, `KB`, `MB`, `GB`, `TB` (for example `512KB`, `10MB`) or plain bytes. **Why are there 4 FFprobe fields?** -- `ffprobe_analyze_duration` + `ffprobe_probe_size` are the default pair for non-live probes (VOD/Series and generic non-live stream probes). -- `ffprobe_live_analyze_duration` + `ffprobe_live_probe_size` are the live-specific pair for all live probes. +- `ffprobe.analyze_duration` + `ffprobe.probe_size` are the default pair for non-live probes (VOD/Series and generic non-live stream probes). +- `ffprobe.live_analyze_duration` + `ffprobe.live_probe_size` are the live-specific pair for all live probes. - This split is intentional because live probing usually needs lower values (less provider load / lower latency), while VOD/Series can use higher values for better metadata extraction quality. @@ -454,6 +506,7 @@ Size fields support `B`, `KB`, `MB`, `GB`, `TB` (for example `512KB`, `10MB`) or - `Resolve task`: A metadata job that fetches or enriches item metadata (for example VOD/Series details, TMDB ID, dates). - `Probe task`: A technical analysis job that inspects stream properties with FFprobe (codec, resolution, audio tracks). +- `TMDB cooldown`: Per-item cooldown that is set when TMDB lookup finished successfully but returned no match. - `Retry`: Re-attempt of a failed task. - `Attempt`: One execution try of a task. If it fails, the attempt counter increases. - `Backoff delay`: Waiting time before the next retry after a failure. @@ -466,6 +519,9 @@ Size fields support `B`, `KB`, `MB`, `GB`, `TB` (for example `512KB`, `10MB`) or - `Cooldown`: Skip period after exhaustion (used for probe tasks) where retries are paused until the cooldown expires. - `Update cycle`: One full metadata processing run for an input, from first queued item until queue idle/completion. - `Resolve exhaustion reset gap`: Time window after which exhausted resolve states are cleared for the next meaningful update cycle. + +Retry/cooldown state is persisted per input in `metadata_retry_state.db` and keeps one bundled record per item key (`resolve`, `probe`, `tmdb`). + - `Pending queue`: In-memory list of metadata tasks waiting to be processed by the worker. - `Worker idle timeout`: Time without immediate work after which a worker may stop and release resources. @@ -1080,10 +1136,6 @@ library: kodi: true jellyfin: false plex: false - tmdb: - enabled: true - # api_key: "4219e299c89411838049ab0dab19ebd5" # Get your API key from https://www.themoviedb.org/settings/api - rate_limit_ms: 250 # Milliseconds between API calls (default: 250ms) fallback_to_filename: true formats: - "nfo" # Optionally write Kodi-compatible NFO files @@ -1092,6 +1144,8 @@ library: series_category: "Local Series" ``` +TMDB resolver settings for library enrichment are configured globally under `metadata_update.tmdb`. + **CLI Usage**: ```bash diff --git a/backend/src/api/endpoints/library_api.rs b/backend/src/api/endpoints/library_api.rs index d0607544b..4336a2adc 100644 --- a/backend/src/api/endpoints/library_api.rs +++ b/backend/src/api/endpoints/library_api.rs @@ -26,16 +26,19 @@ async fn scan_library( }; // Check if Library is enabled - let lib_config = match app_state.app_config.config.load().library.as_ref() { - Some(config) if config.enabled => config.clone(), - _ => { - let response = LibraryScanSummary { - status: LibraryScanSummaryStatus::Error, - message: "Library is not enabled".to_string(), - result: None, - }; - let _ = app_state.event_manager.send_event(EventMessage::LibraryScanProgress(response)); - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Library is not enabled".to_string()}))).into_response(); + let (lib_config, metadata_update_config) = { + let config = app_state.app_config.config.load(); + match config.library.as_ref() { + Some(lib) if lib.enabled => (lib.clone(), config.metadata_update.clone()), + _ => { + let response = LibraryScanSummary { + status: LibraryScanSummaryStatus::Error, + message: "Library is not enabled".to_string(), + result: None, + }; + let _ = app_state.event_manager.send_event(EventMessage::LibraryScanProgress(response)); + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Library is not enabled".to_string()}))).into_response(); + } } }; @@ -44,6 +47,7 @@ async fn scan_library( spawn_library_scan( event_manager, lib_config, + metadata_update_config, client, request.force_rescan, "", @@ -59,11 +63,12 @@ async fn get_library_status( axum::extract::State(app_state): axum::extract::State>, ) -> axum::response::Response { - if let Some(config) = app_state.app_config.config.load().library.as_ref() { + let config_snapshot = app_state.app_config.config.load(); + if let Some(config) = config_snapshot.library.as_ref() { if config.enabled { let client = app_state.http_client.load_full().as_ref().clone(); // Get statistics from processor - let processor = LibraryProcessor::new(config.clone(), client); + let processor = LibraryProcessor::new(config.clone(), config_snapshot.metadata_update.as_ref(), client); let entries = processor.get_all_entries().await; let movies = entries diff --git a/backend/src/api/library_scan.rs b/backend/src/api/library_scan.rs index 904786a3e..3df94a041 100644 --- a/backend/src/api/library_scan.rs +++ b/backend/src/api/library_scan.rs @@ -1,6 +1,6 @@ use crate::api::model::{EventManager, EventMessage, UpdateGuardPermit}; use crate::library::LibraryProcessor; -use crate::model::LibraryConfig; +use crate::model::{LibraryConfig, MetadataUpdateConfig}; use log::{error, info}; use shared::model::{LibraryScanSummary, LibraryScanSummaryStatus}; use std::sync::Arc; @@ -8,6 +8,7 @@ use std::sync::Arc; pub(crate) fn spawn_library_scan( event_manager: Arc, lib_config: LibraryConfig, + metadata_update_config: Option, client: reqwest::Client, force_rescan: bool, message_prefix: &'static str, @@ -16,7 +17,7 @@ pub(crate) fn spawn_library_scan( let prefix = message_prefix.to_string(); tokio::spawn(async move { let _permit = permit; - let processor = LibraryProcessor::new(lib_config, client); + let processor = LibraryProcessor::new(lib_config, metadata_update_config.as_ref(), client); match processor.scan(force_rescan).await { Ok(result) => { info!("{prefix}Library scan completed successfully"); diff --git a/backend/src/api/model/metadata_update_manager.rs b/backend/src/api/model/metadata_update_manager.rs index a01ba7ec0..81be1b184 100644 --- a/backend/src/api/model/metadata_update_manager.rs +++ b/backend/src/api/model/metadata_update_manager.rs @@ -18,9 +18,9 @@ use shared::model::{ }; use shared::utils::generate_playlist_uuid; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, OnceLock, Weak}; use std::time::{Duration, Instant}; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::{mpsc, RwLock, Semaphore}; use tokio_util::sync::CancellationToken; use crate::api::model::BatchResultCollector; @@ -38,10 +38,40 @@ use std::io; use std::path::{Path, PathBuf}; use crate::model::MetadataUpdateConfig; -const PROBE_RETRY_STATE_FILE: &str = "probe_retry_state.db"; +const METADATA_RETRY_STATE_FILE: &str = "metadata_retry_state.db"; const TASK_ERR_NO_CONNECTION: &str = "No connection available"; const TASK_ERR_PREEMPTED: &str = "Task preempted"; const TASK_ERR_UPDATE_IN_PROGRESS: &str = "Playlist update in progress"; +const BLOCKING_DB_MIN_CONCURRENCY: usize = 4; +const BLOCKING_DB_MAX_CONCURRENCY: usize = 32; +const RETRY_STATE_PRUNE_INTERVAL_SECS: i64 = 300; +const RETRY_STATE_MIN_TTL_SECS: i64 = 86_400; +const RUNTIME_SETTINGS_REFRESH_INTERVAL_SECS: u64 = 60; + +fn metadata_blocking_concurrency_limit() -> usize { + let parallelism = + std::thread::available_parallelism().map_or(BLOCKING_DB_MIN_CONCURRENCY, std::num::NonZeroUsize::get); + parallelism + .saturating_mul(2) + .clamp(BLOCKING_DB_MIN_CONCURRENCY, BLOCKING_DB_MAX_CONCURRENCY) +} + +fn metadata_blocking_semaphore() -> &'static Semaphore { + static BLOCKING_SEMAPHORE: OnceLock = OnceLock::new(); + BLOCKING_SEMAPHORE.get_or_init(|| Semaphore::new(metadata_blocking_concurrency_limit())) +} + +async fn spawn_blocking_limited(task: F) -> Result +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + // Throttle B+Tree and file-heavy blocking tasks to avoid saturating Tokio's blocking pool. + let permit = metadata_blocking_semaphore().acquire().await.ok(); + let result = tokio::task::spawn_blocking(task).await; + drop(permit); + result +} #[derive(Debug, Clone)] struct MetadataUpdateRuntimeSettings { @@ -53,8 +83,9 @@ struct MetadataUpdateRuntimeSettings { max_attempts_probe: u8, resolve_exhaustion_reset_gap_secs: i64, probe_cooldown_secs: i64, + tmdb_cooldown_secs: i64, retry_delay_secs: u64, - probe_retry_load_retry_delay_secs: i64, + metadata_retry_load_retry_delay_secs: i64, worker_idle_timeout_secs: u64, max_queue_size: usize, probe_retry_backoff_step_1_secs: u64, @@ -90,22 +121,23 @@ impl MetadataUpdateRuntimeSettings { fn from_metadata_update(cfg: &MetadataUpdateConfig) -> Self { let to_i64 = |v: u64| i64::try_from(v.max(1)).unwrap_or(i64::MAX); Self { - queue_log_interval: Duration::from_secs(cfg.queue_log_interval_secs.max(1)), - progress_log_interval: Duration::from_secs(cfg.progress_log_interval_secs.max(1)), - max_resolve_retry_backoff_secs: cfg.max_resolve_retry_backoff_secs.max(1), - resolve_min_retry_base_secs: cfg.resolve_min_retry_base_secs.max(1), - max_attempts_resolve: cfg.max_attempts_resolve.max(1), - max_attempts_probe: cfg.max_attempts_probe.max(1), - resolve_exhaustion_reset_gap_secs: to_i64(cfg.resolve_exhaustion_reset_gap_secs), - probe_cooldown_secs: to_i64(cfg.probe_cooldown_secs), + queue_log_interval: Duration::from_secs(cfg.log.queue_interval_secs.max(1)), + progress_log_interval: Duration::from_secs(cfg.log.progress_interval_secs.max(1)), + max_resolve_retry_backoff_secs: cfg.resolve.max_retry_backoff_secs.max(1), + resolve_min_retry_base_secs: cfg.resolve.min_retry_base_secs.max(1), + max_attempts_resolve: cfg.resolve.max_attempts.max(1), + max_attempts_probe: cfg.probe.max_attempts.max(1), + resolve_exhaustion_reset_gap_secs: to_i64(cfg.resolve.exhaustion_reset_gap_secs), + probe_cooldown_secs: to_i64(cfg.probe.cooldown_secs), + tmdb_cooldown_secs: to_i64(cfg.tmdb.cooldown_secs), retry_delay_secs: cfg.retry_delay_secs.max(1), - probe_retry_load_retry_delay_secs: to_i64(cfg.probe_retry_load_retry_delay_secs), + metadata_retry_load_retry_delay_secs: to_i64(cfg.probe.retry_load_retry_delay_secs), worker_idle_timeout_secs: cfg.worker_idle_timeout_secs.max(1), max_queue_size: cfg.max_queue_size.max(1), - probe_retry_backoff_step_1_secs: cfg.probe_retry_backoff_step_1_secs.max(1), - probe_retry_backoff_step_2_secs: cfg.probe_retry_backoff_step_2_secs.max(1), - probe_retry_backoff_step_3_secs: cfg.probe_retry_backoff_step_3_secs.max(1), - backoff_jitter_percent: cfg.backoff_jitter_percent.min(95), + probe_retry_backoff_step_1_secs: cfg.probe.retry_backoff_step_1_secs.max(1), + probe_retry_backoff_step_2_secs: cfg.probe.retry_backoff_step_2_secs.max(1), + probe_retry_backoff_step_3_secs: cfg.probe.retry_backoff_step_3_secs.max(1), + backoff_jitter_percent: cfg.probe.backoff_jitter_percent.min(95), } } } @@ -261,28 +293,102 @@ impl RetryState { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RetryDomain { + Resolve, + Probe, + Tmdb, +} + +#[derive(Debug, Clone, Default)] +struct TaskRetryState { + resolve: Option, + probe: Option, + tmdb: Option, + updated_at_ts: i64, +} + +impl TaskRetryState { + fn is_empty(&self) -> bool { self.resolve.is_none() && self.probe.is_none() && self.tmdb.is_none() } + + fn touch(&mut self, now_ts: i64) { + self.updated_at_ts = now_ts.max(1); + } + + fn max_domain_timestamp(&self) -> i64 { + let domain_max = |state: &RetryState| state.next_allowed_at_ts.max(state.cooldown_until_ts.unwrap_or(0)); + self.resolve + .as_ref() + .map_or(0, domain_max) + .max(self.probe.as_ref().map_or(0, domain_max)) + .max(self.tmdb.as_ref().map_or(0, domain_max)) + } + + fn is_stale(&self, now_ts: i64, ttl_secs: i64) -> bool { + let ttl_secs = ttl_secs.max(1); + let anchor_ts = self.updated_at_ts.max(self.max_domain_timestamp()); + now_ts >= anchor_ts.saturating_add(ttl_secs) + } + + fn get(&self, domain: RetryDomain) -> Option<&RetryState> { + match domain { + RetryDomain::Resolve => self.resolve.as_ref(), + RetryDomain::Probe => self.probe.as_ref(), + RetryDomain::Tmdb => self.tmdb.as_ref(), + } + } + + fn get_mut_or_insert(&mut self, domain: RetryDomain) -> &mut RetryState { + let slot = match domain { + RetryDomain::Resolve => &mut self.resolve, + RetryDomain::Probe => &mut self.probe, + RetryDomain::Tmdb => &mut self.tmdb, + }; + slot.get_or_insert_with(RetryState::new) + } + + fn clear_domain(&mut self, domain: RetryDomain) { + match domain { + RetryDomain::Resolve => self.resolve = None, + RetryDomain::Probe => self.probe = None, + RetryDomain::Tmdb => self.tmdb = None, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -enum ProbeRetryDbKey { +enum MetadataRetryDbKey { + VodId(u32), + VodText(String), + SeriesId(u32), + SeriesText(String), LiveId(u32), LiveText(String), Stream { scope: String, id: String }, } -impl ProbeRetryDbKey { - fn from_task_key(task_key: &TaskKey) -> Option { +impl MetadataRetryDbKey { + fn from_task_key(task_key: &TaskKey) -> Self { match task_key { - TaskKey::Live(id) => Some(Self::LiveId(*id)), - TaskKey::LiveStr(id) => Some(Self::LiveText(id.as_ref().to_owned())), - TaskKey::Stream { scope, id } => Some(Self::Stream { + TaskKey::Vod(id) => Self::VodId(*id), + TaskKey::VodStr(id) => Self::VodText(id.as_ref().to_owned()), + TaskKey::Series(id) => Self::SeriesId(*id), + TaskKey::SeriesStr(id) => Self::SeriesText(id.as_ref().to_owned()), + TaskKey::Live(id) => Self::LiveId(*id), + TaskKey::LiveStr(id) => Self::LiveText(id.as_ref().to_owned()), + TaskKey::Stream { scope, id } => Self::Stream { scope: scope.as_ref().to_owned(), id: id.as_ref().to_owned(), - }), - _ => None, + }, } } fn into_task_key(self) -> TaskKey { match self { + Self::VodId(id) => TaskKey::Vod(id), + Self::VodText(id) => TaskKey::VodStr(Arc::from(id)), + Self::SeriesId(id) => TaskKey::Series(id), + Self::SeriesText(id) => TaskKey::SeriesStr(Arc::from(id)), Self::LiveId(id) => TaskKey::Live(id), Self::LiveText(id) => TaskKey::LiveStr(Arc::from(id)), Self::Stream { scope, id } => TaskKey::Stream { @@ -294,32 +400,20 @@ impl ProbeRetryDbKey { } #[derive(Debug, Clone, Serialize, Deserialize)] -struct ProbeRetryDbValue { +struct RetryStateDbValue { attempts: u8, next_allowed_at_ts: i64, cooldown_until_ts: Option, last_error: Option, - updated_at_ts: i64, } -impl ProbeRetryDbValue { - fn from_retry_state(state: &RetryState, updated_at_ts: i64) -> Self { +impl RetryStateDbValue { + fn from_retry_state(state: &RetryState) -> Self { Self { attempts: state.attempts, next_allowed_at_ts: state.next_allowed_at_ts, cooldown_until_ts: state.cooldown_until_ts, last_error: state.last_error.clone(), - updated_at_ts, - } - } - - fn cleared(updated_at_ts: i64) -> Self { - Self { - attempts: 0, - next_allowed_at_ts: 0, - cooldown_until_ts: None, - last_error: None, - updated_at_ts, } } @@ -336,45 +430,125 @@ impl ProbeRetryDbValue { } } -fn ensure_probe_retry_db(path: &Path) -> io::Result<()> { +#[derive(Debug, Clone, Serialize, Deserialize)] +struct MetadataRetryDbValue { + resolve: Option, + probe: Option, + tmdb: Option, + updated_at_ts: i64, +} + +impl MetadataRetryDbValue { + fn from_task_retry_state(state: &TaskRetryState, updated_at_ts: i64) -> Self { + Self { + resolve: state.resolve.as_ref().map(RetryStateDbValue::from_retry_state), + probe: state.probe.as_ref().map(RetryStateDbValue::from_retry_state), + tmdb: state.tmdb.as_ref().map(RetryStateDbValue::from_retry_state), + updated_at_ts, + } + } + + fn cleared(updated_at_ts: i64) -> Self { + Self { + resolve: None, + probe: None, + tmdb: None, + updated_at_ts, + } + } + + fn into_task_retry_state(self) -> Option { + let mut state = TaskRetryState { + resolve: self.resolve.and_then(RetryStateDbValue::into_retry_state), + probe: self.probe.and_then(RetryStateDbValue::into_retry_state), + tmdb: self.tmdb.and_then(RetryStateDbValue::into_retry_state), + updated_at_ts: self.updated_at_ts, + }; + if state.is_empty() { + return None; + } + if state.updated_at_ts <= 0 { + state.updated_at_ts = state.max_domain_timestamp(); + } + Some(state) + } +} + +fn ensure_metadata_retry_db(path: &Path) -> io::Result<()> { if path.exists() { return Ok(()); } - let mut tree = BPlusTree::::new(); + let mut tree = BPlusTree::::new(); tree.store(path).map(|_| ()) } -fn load_probe_retry_states_from_disk(path: &Path) -> io::Result> { - ensure_probe_retry_db(path)?; +fn rebuild_metadata_retry_db(path: &Path, remove_key: Option<&MetadataRetryDbKey>) -> io::Result<()> { + ensure_metadata_retry_db(path)?; + + let mut keep_entries: Vec<(MetadataRetryDbKey, MetadataRetryDbValue)> = Vec::new(); + let mut changed = false; + { + let mut query = BPlusTreeQuery::::try_new(path)?; + for (key, value) in query.iter() { + if remove_key.is_some_and(|target| target == &key) { + changed = true; + continue; + } + if value.clone().into_task_retry_state().is_none() { + changed = true; + continue; + } + keep_entries.push((key, value)); + } + } + + if !changed { + return Ok(()); + } + + let mut tree = BPlusTree::::new(); + for (key, value) in keep_entries { + tree.insert(key, value); + } + tree.store(path).map(|_| ()) +} + +fn load_metadata_retry_states_from_disk(path: &Path) -> io::Result> { + ensure_metadata_retry_db(path)?; let mut result = HashMap::new(); - let mut query = BPlusTreeQuery::::try_new(path)?; + let mut has_tombstones = false; + let mut query = BPlusTreeQuery::::try_new(path)?; for (key, value) in query.iter() { - if let Some(state) = value.into_retry_state() { + if let Some(state) = value.clone().into_task_retry_state() { result.insert(key.into_task_key(), state); + } else { + has_tombstones = true; } } + drop(query); + if has_tombstones { + rebuild_metadata_retry_db(path, None)?; + } Ok(result) } -fn persist_probe_retry_state_to_disk(path: &Path, task_key: &TaskKey, state: Option<&RetryState>) -> io::Result<()> { - let Some(db_key) = ProbeRetryDbKey::from_task_key(task_key) else { - return Ok(()); - }; +fn persist_metadata_retry_state_to_disk(path: &Path, task_key: &TaskKey, state: Option<&TaskRetryState>) -> io::Result<()> { + let db_key = MetadataRetryDbKey::from_task_key(task_key); - ensure_probe_retry_db(path)?; + ensure_metadata_retry_db(path)?; let now_ts = chrono::Utc::now().timestamp(); let value = match state { - Some(s) => ProbeRetryDbValue::from_retry_state(s, now_ts), - None => ProbeRetryDbValue::cleared(now_ts), + Some(s) => MetadataRetryDbValue::from_task_retry_state(s, now_ts), + None => MetadataRetryDbValue::cleared(now_ts), }; - let mut update = BPlusTreeUpdate::::try_new_with_backoff(path)?; + let mut update = BPlusTreeUpdate::::try_new_with_backoff(path)?; update .upsert_batch(&[(&db_key, &value)]) - .map_err(|e| io::Error::other(format!("persist probe retry state failed: {e}")))?; + .map_err(|e| io::Error::other(format!("persist metadata retry state failed: {e}")))?; Ok(()) } @@ -451,7 +625,7 @@ impl MetadataUpdateManager { } /// Acquire exclusive gate for a foreground playlist update. - /// While this guard is held, background metadata/probe tasks are paused. + /// While this guard is held, background workers wait before starting heavy metadata/probe steps. pub async fn acquire_update_pause_guard(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { self.update_pause_gate.clone().write_owned().await } @@ -478,6 +652,7 @@ impl MetadataUpdateManager { /// # Arguments /// * `input_name` - The input this task belongs to /// * `task` - The task to process + #[allow(clippy::too_many_lines)] pub async fn queue_task(&self, input_name: Arc, task: UpdateTask) { // Read app state once and reuse for worker creation when needed. let app_state_weak = { @@ -488,7 +663,8 @@ impl MetadataUpdateManager { let max_queue_size = runtime_settings.max_queue_size; let task_to_queue = task; - for attempt in 0..2 { + let mut channel_closed_attempt: u32 = 0; + loop { // Atomically ensure there is exactly one worker context per input. let mut worker_to_spawn: Option<(u64, InputWorker)> = None; let ctx = match self.workers.entry(input_name.clone()) { @@ -524,9 +700,10 @@ impl MetadataUpdateManager { retry_states: HashMap::new(), resolve_exhausted: HashMap::new(), last_cycle_completed_at_ts: None, - probe_retry_state_path: None, - probe_retry_loaded: false, - probe_retry_load_retry_at_ts: None, + metadata_retry_state_path: None, + metadata_retry_loaded: false, + metadata_retry_load_retry_at_ts: None, + last_retry_state_prune_at_ts: None, scheduled_requeues: Arc::new(DashMap::new()), }, )); @@ -566,17 +743,34 @@ impl MetadataUpdateManager { return; } SubmitTaskResult::ChannelClosed => { + channel_closed_attempt = channel_closed_attempt.saturating_add(1); + if channel_closed_attempt.is_multiple_of(10) { + warn!( + "Metadata enqueue channel still closed for input {input_name} after {channel_closed_attempt} retries; continuing recovery" + ); + } debug_if_enabled!( "Detected closed metadata worker channel for input {}, recreating worker context (attempt {})", input_name, - attempt + 1 + channel_closed_attempt ); Self::remove_worker_context_if_id(&self.workers, &input_name, ctx.worker_id); + let exp = channel_closed_attempt.saturating_sub(1).min(6); + let factor = 1_u64.checked_shl(exp).unwrap_or(u64::MAX); + let backoff_ms = 25_u64.saturating_mul(factor).min(2_000); + let backoff = Duration::from_millis(backoff_ms); + tokio::select! { + () = self.cancel_token.cancelled() => { + warn!( + "Aborting metadata task enqueue for input {input_name} because cancellation was requested: {task_to_queue}" + ); + return; + } + () = tokio::time::sleep(backoff) => {} + } } } } - - warn!("Failed to queue metadata task for input {input_name} after worker recovery attempts: {task_to_queue}"); } fn remove_worker_context_if_id( @@ -600,13 +794,21 @@ impl MetadataUpdateManager { task: UpdateTask, ) -> SubmitTaskResult { let key = TaskKey::from_task(&task); + let task_to_submit = task; if let Some(entry) = pending_tasks.get(&key) { - let mut existing = entry.task.lock(); - if Self::merge_task_payload(&mut existing, task) { - entry.generation.fetch_add(1, Ordering::Relaxed); + if sender.is_closed() { + drop(entry); + if pending_tasks.remove(&key).is_some() { + Self::decrement_pending_task_count(&pending_task_count); + } + } else { + let mut existing = entry.task.lock(); + if Self::merge_task_payload(&mut existing, task_to_submit) { + entry.generation.fetch_add(1, Ordering::Relaxed); + } + return SubmitTaskResult::QueuedOrMerged; } - return SubmitTaskResult::QueuedOrMerged; } // Lock-free admission with CAS: reserve one queue slot only if capacity allows. @@ -623,19 +825,28 @@ impl MetadataUpdateManager { return SubmitTaskResult::QueueFull; } - match pending_tasks.entry(key.clone()) { - Entry::Occupied(entry) => { - // Another producer inserted this key after our fast-path `get`. - // Release reserved capacity and merge into the existing task. - Self::decrement_pending_task_count(&pending_task_count); - let mut existing = entry.get().task.lock(); - if Self::merge_task_payload(&mut existing, task) { - entry.get().generation.fetch_add(1, Ordering::Relaxed); + loop { + match pending_tasks.entry(key.clone()) { + Entry::Occupied(entry) => { + // Another producer inserted this key after our fast-path `get`. + if sender.is_closed() { + entry.remove(); + Self::decrement_pending_task_count(&pending_task_count); + continue; + } + + // Release reserved capacity and merge into the existing task. + Self::decrement_pending_task_count(&pending_task_count); + let mut existing = entry.get().task.lock(); + if Self::merge_task_payload(&mut existing, task_to_submit) { + entry.get().generation.fetch_add(1, Ordering::Relaxed); + } + return SubmitTaskResult::QueuedOrMerged; + } + Entry::Vacant(entry) => { + entry.insert(PendingTask::new(task_to_submit)); + break; } - return SubmitTaskResult::QueuedOrMerged; - } - Entry::Vacant(entry) => { - entry.insert(PendingTask::new(task)); } } @@ -658,7 +869,7 @@ impl MetadataUpdateManager { } fn merge_task_payload(existing: &mut UpdateTask, task: UpdateTask) -> bool { - let mut merged = false; + let mut changed = false; // Merge logic match (existing, task) { ( @@ -669,9 +880,11 @@ impl MetadataUpdateManager { UpdateTask::ResolveSeries { reason: r1, delay: d1, .. }, UpdateTask::ResolveSeries { reason: r2, delay: d2, .. }, ) => { + let previous_reason = *r1; + let previous_delay = *d1; *r1 |= r2; *d1 = min(*d1, d2); - merged = true; + changed = *r1 != previous_reason || *d1 != previous_delay; } ( UpdateTask::ProbeStream { @@ -689,6 +902,10 @@ impl MetadataUpdateManager { .. }, ) => { + let previous_reason = *r1; + let previous_delay = *d1; + let previous_url = url1.clone(); + let previous_item_type = *item_type1; *r1 |= r2; *d1 = min(*d1, d2); // Keep the existing payload by default; only fill it from the incoming @@ -697,21 +914,27 @@ impl MetadataUpdateManager { *url1 = url2; *item_type1 = item_type2; } - merged = true; + changed = *r1 != previous_reason + || *d1 != previous_delay + || *url1 != previous_url + || *item_type1 != previous_item_type; } ( UpdateTask::ProbeLive { reason: r1, delay: d1, interval: i1, .. }, UpdateTask::ProbeLive { reason: r2, delay: d2, interval: i2, .. }, ) => { + let previous_reason = *r1; + let previous_delay = *d1; + let previous_interval = *i1; *r1 |= r2; *d1 = min(*d1, d2); *i1 = min(*i1, i2); - merged = true; + changed = *r1 != previous_reason || *d1 != previous_delay || *i1 != previous_interval; } _ => {} // Mismatched types, should not happen due to TaskKey } - merged + changed } /// Queue a task using the legacy API (for backward compatibility). @@ -743,17 +966,24 @@ struct InputWorker { batch_buffer: BatchResultCollector, db_handles: HashMap, failed_clusters: HashSet, - retry_states: HashMap, + retry_states: HashMap, resolve_exhausted: HashMap, last_cycle_completed_at_ts: Option, - probe_retry_state_path: Option, - probe_retry_loaded: bool, - probe_retry_load_retry_at_ts: Option, + metadata_retry_state_path: Option, + metadata_retry_loaded: bool, + metadata_retry_load_retry_at_ts: Option, + last_retry_state_prune_at_ts: Option, // Shared with detached delayed requeue tasks spawned in `schedule_requeue_at`. // A plain HashMap cannot be moved safely into those `'static` tasks. scheduled_requeues: Arc>, } +#[derive(Debug, Clone, Copy)] +struct ProcessTaskOutcome { + task_changed: bool, + tmdb_pending: bool, +} + impl InputWorker { #[allow(clippy::too_many_lines)] async fn run(mut self) { @@ -768,15 +998,21 @@ impl InputWorker { let input_name = self.input_name.clone(); let app_state_weak = self.app_state_weak.clone(); - let startup_runtime_settings = self.runtime_settings(); - self.ensure_probe_retry_state_loaded(&input_name, app_state_weak.as_ref(), &startup_runtime_settings) + let mut runtime_settings = self.runtime_settings(); + let mut last_runtime_settings_refresh_at = Instant::now(); + self.ensure_metadata_retry_state_loaded(&input_name, app_state_weak.as_ref(), &runtime_settings) .await; // Keep one prefetched task to minimize channel waits/lock churn. let mut next_task: Option<(TaskKey, UpdateTask, u64)> = None; loop { - let runtime_settings = self.runtime_settings(); + if last_runtime_settings_refresh_at.elapsed() + >= Duration::from_secs(RUNTIME_SETTINGS_REFRESH_INTERVAL_SECS) + { + runtime_settings = self.runtime_settings(); + last_runtime_settings_refresh_at = Instant::now(); + } let task_data = if let Some(t) = next_task.take() { Some(t) } else { @@ -784,11 +1020,12 @@ impl InputWorker { }; let Some((current_key, current_task, current_generation)) = task_data else { break }; - if !self.probe_retry_loaded { - self.ensure_probe_retry_state_loaded(&input_name, app_state_weak.as_ref(), &runtime_settings) + if !self.metadata_retry_loaded { + self.ensure_metadata_retry_state_loaded(&input_name, app_state_weak.as_ref(), &runtime_settings) .await; } let now_ts = chrono::Utc::now().timestamp(); + self.prune_retry_tracking_maps_if_needed(now_ts, &runtime_settings).await; if !queue_cycle_active { // First entry of a new processing cycle. @@ -816,14 +1053,15 @@ impl InputWorker { let mut schedule_requeue_at_ts: Option = None; let mut remove_current_task = false; let mut apply_rate_limit = false; - let mut probe_persist_state: Option> = None; + let mut metadata_persist_state: Option> = None; let mut skip_execution = false; + let mut task_for_execution = current_task.clone(); - if Self::is_resolve_task(¤t_task) && self.resolve_exhausted.contains_key(¤t_key) { + if Self::is_resolve_task(&task_for_execution) && self.resolve_exhausted.contains_key(¤t_key) { debug_if_enabled!( "Skipping exhausted resolve task for input {}: {} (reset window: {}s)", input_name, - current_task, + task_for_execution, runtime_settings.resolve_exhaustion_reset_gap_secs ); self.scheduled_requeues.remove(¤t_key); @@ -832,52 +1070,104 @@ impl InputWorker { } if !skip_execution { - let mut clear_probe_state = false; - if let Some(state) = self.retry_states.get(¤t_key) { - if let Some(cooldown_until_ts) = state.cooldown_until_ts { - if now_ts < cooldown_until_ts { - debug_if_enabled!( - "Skipping probe task in cooldown for input {}: {} (cooldown_until={})", - input_name, - current_task, - cooldown_until_ts - ); - self.scheduled_requeues.remove(¤t_key); - remove_current_task = true; - skip_execution = true; - } else if Self::is_probe_task(¤t_task) { - clear_probe_state = true; + let mut clear_tmdb_state = false; + if let Some(state_bundle) = self.retry_states.get(¤t_key) { + if let Some(tmdb_state) = state_bundle.get(RetryDomain::Tmdb) { + if let Some(cooldown_until_ts) = tmdb_state.cooldown_until_ts { + if now_ts < cooldown_until_ts { + if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) { + debug_if_enabled!( + "TMDB cooldown active for input {}: {} (cooldown_until={}), continuing with non-TMDB reasons", + input_name, + task_for_execution, + cooldown_until_ts + ); + task_for_execution = stripped_task; + } else { + debug_if_enabled!( + "Skipping TMDB-only task in cooldown for input {}: {} (cooldown_until={})", + input_name, + task_for_execution, + cooldown_until_ts + ); + self.scheduled_requeues.remove(¤t_key); + remove_current_task = true; + skip_execution = true; + } + } else { + clear_tmdb_state = true; + } } } - - if !skip_execution && state.next_allowed_at_ts > now_ts { - schedule_requeue_at_ts = Some(state.next_allowed_at_ts); - skip_execution = true; - } } - if clear_probe_state { - self.retry_states.remove(¤t_key); - probe_persist_state = Some(None); + let active_retry_domain = Self::retry_domain_for_task(&task_for_execution); + let mut clear_active_retry_state = false; + + if !skip_execution { + if let Some(state_bundle) = self.retry_states.get(¤t_key) { + if let Some(active_state) = state_bundle.get(active_retry_domain) { + if active_retry_domain == RetryDomain::Probe { + if let Some(cooldown_until_ts) = active_state.cooldown_until_ts { + if now_ts < cooldown_until_ts { + debug_if_enabled!( + "Skipping probe task in cooldown for input {}: {} (cooldown_until={})", + input_name, + task_for_execution, + cooldown_until_ts + ); + self.scheduled_requeues.remove(¤t_key); + remove_current_task = true; + skip_execution = true; + } else { + clear_active_retry_state = true; + } + } + } + + if !skip_execution && active_state.next_allowed_at_ts > now_ts { + schedule_requeue_at_ts = Some(active_state.next_allowed_at_ts); + skip_execution = true; + } + } + } + } + + if clear_active_retry_state || clear_tmdb_state { + let mut should_remove_retry_entry = false; + let mut state_after_clear: Option = None; + + if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { + if clear_active_retry_state { + state_bundle.clear_domain(active_retry_domain); + } + if clear_tmdb_state { + state_bundle.clear_domain(RetryDomain::Tmdb); + } + if state_bundle.is_empty() { + should_remove_retry_entry = true; + } else { + state_bundle.touch(now_ts); + state_after_clear = Some(state_bundle.clone()); + } + } + + if should_remove_retry_entry { + self.retry_states.remove(¤t_key); + } + metadata_persist_state = Some(state_after_clear); } } if !skip_execution { - // Hold READ gate while executing one background task. - // Foreground updates acquire WRITE gate and therefore pause this path. let task_result = { - let _task_gate_guard = if let Ok(guard) = self.update_pause_gate.clone().try_read_owned() { - guard - } else { - // Foreground update contention detected. - // Drop cached file read handles before blocking on the gate to avoid AB-BA deadlocks. - self.release_db_handles(); - self.update_pause_gate.clone().read_owned().await + let Some(_pause_guard) = self.wait_for_update_pause_window().await else { + break; }; Self::process_task_static( &input_name, app_state_weak.as_ref(), - ¤t_task, + &task_for_execution, &mut self.batch_buffer, &mut self.db_handles, &mut self.failed_clusters, @@ -886,19 +1176,83 @@ impl InputWorker { }; match task_result { - Ok(task_changed) => { + Ok(task_outcome) => { if Self::is_vod_task_key(¤t_key) { processed_vod_count += 1; } else if Self::is_series_task_key(¤t_key) { processed_series_count += 1; } - debug!("Processed metadata task for input {input_name}: {current_task} (changed={task_changed})"); - self.retry_states.remove(¤t_key); + debug!( + "Processed metadata task for input {input_name}: {task_for_execution} (changed={}, tmdb_pending={})", + task_outcome.task_changed, + task_outcome.tmdb_pending + ); + + let active_retry_domain = Self::retry_domain_for_task(&task_for_execution); + let task_has_tmdb_reason = Self::task_has_tmdb_reason(&task_for_execution); + let mut should_remove_retry_entry = false; + let mut state_after_success: Option = None; + + if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { + state_bundle.clear_domain(active_retry_domain); + if task_has_tmdb_reason { + if task_outcome.tmdb_pending { + let cooldown_until_ts = now_ts.saturating_add(runtime_settings.tmdb_cooldown_secs); + let tmdb_state = state_bundle.get_mut_or_insert(RetryDomain::Tmdb); + tmdb_state.attempts = 0; + tmdb_state.next_allowed_at_ts = cooldown_until_ts; + tmdb_state.cooldown_until_ts = Some(cooldown_until_ts); + tmdb_state.last_error = + Some("TMDB lookup completed without matching result".to_string()); + debug_if_enabled!( + "TMDB resolve produced no match for input {}: {} (cooldown_until={})", + input_name, + task_for_execution, + cooldown_until_ts + ); + } else { + state_bundle.clear_domain(RetryDomain::Tmdb); + } + } + + if state_bundle.is_empty() { + should_remove_retry_entry = true; + } else { + state_bundle.touch(now_ts); + state_after_success = Some(state_bundle.clone()); + } + } else if task_has_tmdb_reason && task_outcome.tmdb_pending { + let cooldown_until_ts = now_ts.saturating_add(runtime_settings.tmdb_cooldown_secs); + let state_bundle = TaskRetryState { + resolve: None, + probe: None, + tmdb: Some(RetryState { + attempts: 0, + next_allowed_at_ts: cooldown_until_ts, + cooldown_until_ts: Some(cooldown_until_ts), + last_error: Some("TMDB lookup completed without matching result".to_string()), + }), + updated_at_ts: now_ts.max(1), + }; + self.retry_states.insert(current_key.clone(), state_bundle.clone()); + state_after_success = Some(state_bundle); + debug_if_enabled!( + "TMDB resolve produced no match for input {}: {} (cooldown_until={})", + input_name, + task_for_execution, + cooldown_until_ts + ); + } + + if should_remove_retry_entry { + self.retry_states.remove(¤t_key); + } + if should_remove_retry_entry || state_after_success.is_some() { + metadata_persist_state = Some(state_after_success); + } + self.resolve_exhausted.remove(¤t_key); self.scheduled_requeues.remove(¤t_key); - if Self::is_probe_task(¤t_task) { - probe_persist_state = Some(None); - } if last_progress_log_at.elapsed() >= runtime_settings.progress_log_interval { // current_key is removed from pending_tasks later in this loop iteration; @@ -923,7 +1277,54 @@ impl InputWorker { apply_rate_limit = true; } Err(e) => { - if Self::is_transient_worker_error(&e.message) { + if Self::is_permanent_not_found_error(&e.message) { + let retry_domain = Self::retry_domain_for_task(&task_for_execution); + self.scheduled_requeues.remove(¤t_key); + if retry_domain == RetryDomain::Probe { + let cooldown_until_ts = now_ts.saturating_add(runtime_settings.probe_cooldown_secs); + let state_bundle_after_update = { + let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); + let state = state_bundle.get_mut_or_insert(RetryDomain::Probe); + state.attempts = runtime_settings.max_attempts_probe; + state.next_allowed_at_ts = cooldown_until_ts; + state.cooldown_until_ts = Some(cooldown_until_ts); + state.last_error = Some(e.message.clone()); + state_bundle.touch(now_ts); + state_bundle.clone() + }; + metadata_persist_state = Some(Some(state_bundle_after_update)); + remove_current_task = true; + debug_if_enabled!( + "Probe task entered cooldown after permanent not-found for input {}: {} (cooldown_until={})", + input_name, + task_for_execution, + cooldown_until_ts + ); + } else { + self.resolve_exhausted.insert(current_key.clone(), now_ts); + let mut should_remove_retry_entry = false; + let mut state_after_clear: Option = None; + if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { + state_bundle.clear_domain(retry_domain); + if state_bundle.is_empty() { + should_remove_retry_entry = true; + } else { + state_bundle.touch(now_ts); + state_after_clear = Some(state_bundle.clone()); + } + } + if should_remove_retry_entry { + self.retry_states.remove(¤t_key); + } + metadata_persist_state = Some(state_after_clear); + remove_current_task = true; + debug_if_enabled!( + "Resolve task marked exhausted after permanent not-found for input {}: {}", + input_name, + task_for_execution + ); + } + } else if Self::is_transient_worker_error(&e.message) { if e.message == TASK_ERR_UPDATE_IN_PROGRESS { // Drop cached readers quickly so foreground writer can progress. self.release_db_handles(); @@ -936,82 +1337,96 @@ impl InputWorker { debug_if_enabled!( "Transient task deferral for input {}: {} (retry_in={}s, err={})", input_name, - current_task, + task_for_execution, retry_delay_secs, e.message ); } else { - let is_probe_task = Self::is_probe_task(¤t_task); - let max_attempts = if is_probe_task { + let retry_domain = Self::retry_domain_for_task(&task_for_execution); + let max_attempts = if retry_domain == RetryDomain::Probe { runtime_settings.max_attempts_probe } else { runtime_settings.max_attempts_resolve }; - let state_after_update = { - let state = self - .retry_states - .entry(current_key.clone()) - .or_insert_with(RetryState::new); - state.attempts = state.attempts.saturating_add(1); - state.last_error = Some(e.message.clone()); + let (state_after_update, state_bundle_after_update) = { + let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); + let state_after_update = { + let state = state_bundle.get_mut_or_insert(retry_domain); + state.attempts = state.attempts.saturating_add(1); + state.last_error = Some(e.message.clone()); - if state.attempts < max_attempts { - let backoff_secs = if is_probe_task { - Self::compute_probe_retry_backoff_secs(state.attempts, &runtime_settings) - } else { - Self::compute_resolve_retry_backoff_secs( - current_task.delay(), - state.attempts, - &runtime_settings, - ) - }; - let backoff_i64 = i64::try_from(backoff_secs).unwrap_or(i64::MAX); - state.next_allowed_at_ts = now_ts.saturating_add(backoff_i64); - state.cooldown_until_ts = None; - } else if is_probe_task { - state.cooldown_until_ts = - Some(now_ts.saturating_add(runtime_settings.probe_cooldown_secs)); - state.next_allowed_at_ts = state.cooldown_until_ts.unwrap_or(now_ts); - } + if state.attempts < max_attempts { + let backoff_secs = if retry_domain == RetryDomain::Probe { + Self::compute_probe_retry_backoff_secs(state.attempts, &runtime_settings) + } else { + Self::compute_resolve_retry_backoff_secs( + task_for_execution.delay(), + state.attempts, + &runtime_settings, + ) + }; + let backoff_i64 = i64::try_from(backoff_secs).unwrap_or(i64::MAX); + state.next_allowed_at_ts = now_ts.saturating_add(backoff_i64); + state.cooldown_until_ts = None; + } else if retry_domain == RetryDomain::Probe { + state.cooldown_until_ts = + Some(now_ts.saturating_add(runtime_settings.probe_cooldown_secs)); + state.next_allowed_at_ts = state.cooldown_until_ts.unwrap_or(now_ts); + } + state.clone() + }; + state_bundle.touch(now_ts); - state.clone() + (state_after_update, state_bundle.clone()) }; let attempts = state_after_update.attempts; if attempts >= max_attempts { self.scheduled_requeues.remove(¤t_key); - if is_probe_task { + if retry_domain == RetryDomain::Probe { remove_current_task = true; - probe_persist_state = Some(Some(state_after_update.clone())); + metadata_persist_state = Some(Some(state_bundle_after_update.clone())); debug_if_enabled!( "Probe task exhausted for input {}: {} (attempts={}, cooldown_until={:?})", input_name, - current_task, + task_for_execution, state_after_update.attempts, state_after_update.cooldown_until_ts ); } else { self.resolve_exhausted.insert(current_key.clone(), now_ts); - self.retry_states.remove(¤t_key); + let mut should_remove_retry_entry = false; + let mut state_after_clear: Option = None; + if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { + state_bundle.clear_domain(retry_domain); + if state_bundle.is_empty() { + should_remove_retry_entry = true; + } else { + state_bundle.touch(now_ts); + state_after_clear = Some(state_bundle.clone()); + } + } + if should_remove_retry_entry { + self.retry_states.remove(¤t_key); + } + metadata_persist_state = Some(state_after_clear); remove_current_task = true; debug_if_enabled!( "Resolve task exhausted for input {}: {} (attempts={})", input_name, - current_task, + task_for_execution, attempts ); } } else { schedule_requeue_at_ts = Some(state_after_update.next_allowed_at_ts); - if is_probe_task { - probe_persist_state = Some(Some(state_after_update.clone())); - } + metadata_persist_state = Some(Some(state_bundle_after_update)); debug_if_enabled!( "Task failed for input {}, scheduling retry: {} (attempt={}, next_allowed_at={}, err={})", input_name, - current_task, + task_for_execution, attempts, state_after_update.next_allowed_at_ts, e.message @@ -1022,14 +1437,16 @@ impl InputWorker { } } - if let Some(state) = probe_persist_state { - self.persist_probe_retry_state(¤t_key, state.as_ref()).await; + if let Some(state) = metadata_persist_state { + self.persist_metadata_retry_state(¤t_key, state.as_ref()).await; } // Check and flush batch if self.batch_buffer.should_flush() { self.release_db_handles(); - let _gate_guard = self.update_pause_gate.clone().read_owned().await; + let Some(_pause_guard) = self.wait_for_update_pause_window().await else { + break; + }; Self::flush_batch_static(&input_name, app_state_weak.as_ref(), &mut self.batch_buffer).await; } @@ -1080,7 +1497,9 @@ impl InputWorker { // If no immediate work is available, flush buffered results now even when delayed retries remain pending. if !channel_has_work && !self.batch_buffer.is_empty() { self.release_db_handles(); - let _gate_guard = self.update_pause_gate.clone().read_owned().await; + let Some(_pause_guard) = self.wait_for_update_pause_window().await else { + break; + }; Self::flush_batch_static(&input_name, app_state_weak.as_ref(), &mut self.batch_buffer).await; } @@ -1101,48 +1520,49 @@ impl InputWorker { // Final flush self.release_db_handles(); if !self.batch_buffer.is_empty() { - let _gate_guard = self.update_pause_gate.clone().read_owned().await; - Self::flush_batch_static(&input_name, app_state_weak.as_ref(), &mut self.batch_buffer).await; + if let Some(_pause_guard) = self.wait_for_update_pause_window().await { + Self::flush_batch_static(&input_name, app_state_weak.as_ref(), &mut self.batch_buffer).await; + } } debug!("Metadata worker stopped for input {input_name}"); } - async fn ensure_probe_retry_state_loaded( + async fn ensure_metadata_retry_state_loaded( &mut self, input_name: &str, app_state_weak: Option<&Weak>, runtime_settings: &MetadataUpdateRuntimeSettings, ) { - if self.probe_retry_loaded { + if self.metadata_retry_loaded { return; } let now_ts = chrono::Utc::now().timestamp(); if self - .probe_retry_load_retry_at_ts + .metadata_retry_load_retry_at_ts .is_some_and(|retry_at_ts| now_ts < retry_at_ts) { return; } let Some(app_state) = app_state_weak.and_then(Weak::upgrade) else { - self.probe_retry_load_retry_at_ts = - Some(now_ts.saturating_add(runtime_settings.probe_retry_load_retry_delay_secs)); + self.metadata_retry_load_retry_at_ts = + Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); return; }; let working_dir = app_state.app_config.config.load().working_dir.clone(); let Ok(storage_path) = get_input_storage_path(input_name, &working_dir).await else { - warn!("Could not resolve storage path for probe retry state on input {input_name}"); - self.probe_retry_load_retry_at_ts = - Some(now_ts.saturating_add(runtime_settings.probe_retry_load_retry_delay_secs)); + warn!("Could not resolve storage path for metadata retry state on input {input_name}"); + self.metadata_retry_load_retry_at_ts = + Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); return; }; - let retry_path = storage_path.join(PROBE_RETRY_STATE_FILE); - self.probe_retry_state_path = Some(retry_path.clone()); + let retry_path = storage_path.join(METADATA_RETRY_STATE_FILE); + self.metadata_retry_state_path = Some(retry_path.clone()); - let loaded = tokio::task::spawn_blocking(move || load_probe_retry_states_from_disk(&retry_path)) + let loaded = spawn_blocking_limited(move || load_metadata_retry_states_from_disk(&retry_path)) .await .map_err(|err| err.to_string()) .and_then(|result| result.map_err(|err| err.to_string())); @@ -1150,41 +1570,43 @@ impl InputWorker { let loaded = match loaded { Ok(states) => states, Err(err) => { - warn!("Failed to load probe retry state for input {input_name}: {err}"); - self.probe_retry_load_retry_at_ts = - Some(now_ts.saturating_add(runtime_settings.probe_retry_load_retry_delay_secs)); + warn!("Failed to load metadata retry state for input {input_name}: {err}"); + self.metadata_retry_load_retry_at_ts = + Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); return; } }; - // Intentionally do not resurrect pending probe tasks solely from persisted retry state. + // Intentionally do not resurrect pending tasks solely from persisted retry state. // The persisted state is applied once the corresponding task is naturally queued again // (for example by the next playlist update), because state alone does not contain the // full `UpdateTask` payload for all variants. for (key, state) in loaded { self.retry_states.insert(key, state); } - self.probe_retry_loaded = true; - self.probe_retry_load_retry_at_ts = None; + self.prune_retry_tracking_maps(now_ts, runtime_settings).await; + self.last_retry_state_prune_at_ts = Some(now_ts); + self.metadata_retry_loaded = true; + self.metadata_retry_load_retry_at_ts = None; } - async fn persist_probe_retry_state(&self, key: &TaskKey, state: Option<&RetryState>) { - let Some(path) = self.probe_retry_state_path.clone() else { + async fn persist_metadata_retry_state(&self, key: &TaskKey, state: Option<&TaskRetryState>) { + let Some(path) = self.metadata_retry_state_path.clone() else { return; }; let key = key.clone(); let state = state.cloned(); let input_name = self.input_name.clone(); - let persist_result = tokio::task::spawn_blocking(move || { - persist_probe_retry_state_to_disk(&path, &key, state.as_ref()) + let persist_result = spawn_blocking_limited(move || { + persist_metadata_retry_state_to_disk(&path, &key, state.as_ref()) }) .await; match persist_result { Ok(Ok(())) => {} - Ok(Err(err)) => warn!("Failed to persist probe retry state for input {input_name}: {err}"), - Err(err) => warn!("Failed to persist probe retry state for input {input_name}: {err}"), + Ok(Err(err)) => warn!("Failed to persist metadata retry state for input {input_name}: {err}"), + Err(err) => warn!("Failed to persist metadata retry state for input {input_name}: {err}"), } } @@ -1338,6 +1760,92 @@ impl InputWorker { MetadataUpdateRuntimeSettings::from_app_state(self.app_state_weak.as_ref()) } + async fn wait_for_update_pause_window(&mut self) -> Option> { + if let Ok(guard) = self.update_pause_gate.clone().try_read_owned() { + return Some(guard); + } + + // Foreground writer is active or queued. Release cached handles before waiting to avoid AB-BA patterns. + self.release_db_handles(); + tokio::select! { + () = self.cancel_token.cancelled() => None, + guard = self.update_pause_gate.clone().read_owned() => { + Some(guard) + } + } + } + + fn retry_state_ttl_secs(runtime_settings: &MetadataUpdateRuntimeSettings) -> i64 { + let max_resolve_backoff = i64::try_from(runtime_settings.max_resolve_retry_backoff_secs).unwrap_or(i64::MAX); + runtime_settings + .tmdb_cooldown_secs + .max(runtime_settings.probe_cooldown_secs) + .max(runtime_settings.resolve_exhaustion_reset_gap_secs) + .max(max_resolve_backoff) + .saturating_mul(6) + .max(RETRY_STATE_MIN_TTL_SECS) + } + + fn resolve_exhausted_ttl_secs(runtime_settings: &MetadataUpdateRuntimeSettings) -> i64 { + runtime_settings + .resolve_exhaustion_reset_gap_secs + .saturating_mul(6) + .max(RETRY_STATE_MIN_TTL_SECS) + } + + async fn prune_retry_tracking_maps_if_needed( + &mut self, + now_ts: i64, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) { + let due = self + .last_retry_state_prune_at_ts + .is_none_or(|last| now_ts.saturating_sub(last) >= RETRY_STATE_PRUNE_INTERVAL_SECS); + if !due { + return; + } + self.last_retry_state_prune_at_ts = Some(now_ts); + self.prune_retry_tracking_maps(now_ts, runtime_settings).await; + } + + async fn prune_retry_tracking_maps(&mut self, now_ts: i64, runtime_settings: &MetadataUpdateRuntimeSettings) { + let retry_state_ttl_secs = Self::retry_state_ttl_secs(runtime_settings); + let resolve_exhausted_ttl_secs = Self::resolve_exhausted_ttl_secs(runtime_settings); + let stale_retry_keys: Vec = self + .retry_states + .iter() + .filter_map(|(key, state)| { + if state.is_stale(now_ts, retry_state_ttl_secs) { + Some(key.clone()) + } else { + None + } + }) + .collect(); + + for key in stale_retry_keys { + if self.retry_states.remove(&key).is_some() { + debug_if_enabled!("Pruned stale metadata retry state for input {}: {:?}", self.input_name, key); + self.persist_metadata_retry_state(&key, None).await; + } + } + + let stale_resolve_exhausted_keys: Vec = self + .resolve_exhausted + .iter() + .filter_map(|(key, exhausted_at_ts)| { + if now_ts.saturating_sub(*exhausted_at_ts) >= resolve_exhausted_ttl_secs { + Some(key.clone()) + } else { + None + } + }) + .collect(); + for key in stale_resolve_exhausted_keys { + self.resolve_exhausted.remove(&key); + } + } + fn release_db_handles(&mut self) { if !self.db_handles.is_empty() { self.db_handles.clear(); @@ -1419,17 +1927,118 @@ impl InputWorker { matches!(task, UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. }) } + #[inline] + fn is_probe_only_resolve_task(task: &UpdateTask) -> bool { + match task { + UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { + reason.contains(ResolveReason::Probe) + && !reason.contains(ResolveReason::Info) + && !reason.contains(ResolveReason::Tmdb) + && !reason.contains(ResolveReason::Date) + } + _ => false, + } + } + + #[inline] + fn retry_domain_for_task(task: &UpdateTask) -> RetryDomain { + if Self::is_probe_task(task) || Self::is_probe_only_resolve_task(task) { + RetryDomain::Probe + } else { + RetryDomain::Resolve + } + } + #[inline] fn is_resolve_task(task: &UpdateTask) -> bool { matches!(task, UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. }) } + #[inline] + fn task_has_tmdb_reason(task: &UpdateTask) -> bool { + match task { + UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { + reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) + } + _ => false, + } + } + + fn strip_tmdb_reasons(task: &UpdateTask) -> Option { + match task { + UpdateTask::ResolveVod { id, reason, delay } => { + let mut next_reason = *reason; + next_reason.unset(ResolveReason::Tmdb); + next_reason.unset(ResolveReason::Date); + if next_reason.is_empty() { + None + } else { + Some(UpdateTask::ResolveVod { + id: id.clone(), + reason: next_reason, + delay: *delay, + }) + } + } + UpdateTask::ResolveSeries { id, reason, delay } => { + let mut next_reason = *reason; + next_reason.unset(ResolveReason::Tmdb); + next_reason.unset(ResolveReason::Date); + if next_reason.is_empty() { + None + } else { + Some(UpdateTask::ResolveSeries { + id: id.clone(), + reason: next_reason, + delay: *delay, + }) + } + } + _ => Some(task.clone()), + } + } + fn is_transient_worker_error(message: &str) -> bool { message == TASK_ERR_UPDATE_IN_PROGRESS || message == TASK_ERR_PREEMPTED || message == TASK_ERR_NO_CONNECTION } + #[inline] + fn is_permanent_not_found_error(message: &str) -> bool { + let normalized = message.to_ascii_lowercase(); + Self::contains_standalone_fragment(&normalized, "404") + || Self::contains_standalone_fragment(&normalized, "not found") + } + + #[inline] + fn is_word_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' + } + + fn contains_standalone_fragment(haystack: &str, fragment: &str) -> bool { + if fragment.is_empty() || haystack.len() < fragment.len() { + return false; + } + + let bytes = haystack.as_bytes(); + let mut search_from = 0usize; + + while let Some(relative_idx) = haystack[search_from..].find(fragment) { + let start = search_from + relative_idx; + let end = start + fragment.len(); + let before_is_word = start > 0 && Self::is_word_byte(bytes[start - 1]); + let after_is_word = end < bytes.len() && Self::is_word_byte(bytes[end]); + + if !before_is_word && !after_is_word { + return true; + } + search_from = end; + } + + false + } + // Changed to static method async fn flush_batch_static( input_name: &str, @@ -1571,7 +2180,7 @@ impl InputWorker { // Scope read lock strictly to mapping load. let _file_lock = app_state.app_config.file_locks.read_lock(&mapping_file).await; let mapping_file_clone = mapping_file.clone(); - match tokio::task::spawn_blocking(move || { + match spawn_blocking_limited(move || { TargetIdMapping::new(&mapping_file_clone, false) }) .await @@ -1764,7 +2373,7 @@ impl InputWorker { // Scope read lock to read-only query phase so write phase can acquire lock. let _file_lock = app_state.app_config.file_locks.read_lock(&xtream_path).await; let xtream_path_clone = xtream_path.clone(); - match tokio::task::spawn_blocking(move || -> Vec { + match spawn_blocking_limited(move || -> Vec { let mut updates = Vec::with_capacity(updates_input.len()); let Ok(mut query) = BPlusTreeQuery::::try_new(&xtream_path_clone) else { return updates; @@ -1822,7 +2431,7 @@ impl InputWorker { // Scope read lock to read-only query phase so write phase can acquire lock. let _file_lock = app_state.app_config.file_locks.read_lock(&xtream_path).await; let xtream_path_clone = xtream_path.clone(); - match tokio::task::spawn_blocking(move || -> Vec { + match spawn_blocking_limited(move || -> Vec { let mut updates = Vec::with_capacity(updates_input.len()); let Ok(mut query) = BPlusTreeQuery::::try_new(&xtream_path_clone) else { return updates; @@ -1880,7 +2489,7 @@ impl InputWorker { // Scope read lock to read-only query phase so write phase can acquire lock. let _file_lock = app_state.app_config.file_locks.read_lock(&xtream_path).await; let xtream_path_clone = xtream_path.clone(); - match tokio::task::spawn_blocking(move || -> Vec { + match spawn_blocking_limited(move || -> Vec { let mut updates = Vec::with_capacity(updates_input.len()); let Ok(mut query) = BPlusTreeQuery::::try_new(&xtream_path_clone) else { return updates; @@ -1957,7 +2566,7 @@ impl InputWorker { if file_path.exists() { let lock = app_state.app_config.file_locks.read_lock(&file_path).await; let file_path = file_path.clone(); - let query = match tokio::task::spawn_blocking(move || { + let query = match spawn_blocking_limited(move || { BPlusTreeQuery::::try_new(&file_path) }) .await @@ -2008,7 +2617,7 @@ impl InputWorker { Self::get_or_open_query(input_name, app_state, cluster, db_handles, failed_clusters).await { let query = Arc::clone(&query); - let item = match tokio::task::spawn_blocking(move || { + let item = match spawn_blocking_limited(move || { let mut guard = query.lock(); guard.query_zero_copy(&stream_id).ok().flatten() }) @@ -2029,6 +2638,19 @@ impl InputWorker { None } + fn vod_tmdb_pending_for_reason(props: &VideoStreamProperties, reason: ResolveReasonSet) -> bool { + let tmdb_missing = reason.contains(ResolveReason::Tmdb) && props.tmdb.is_none(); + let date_missing = + reason.contains(ResolveReason::Date) && props.details.as_ref().and_then(|d| d.release_date.as_ref()).is_none(); + tmdb_missing || date_missing + } + + fn series_tmdb_pending_for_reason(props: &SeriesStreamProperties, reason: ResolveReasonSet) -> bool { + let tmdb_missing = reason.contains(ResolveReason::Tmdb) && props.tmdb.is_none(); + let date_missing = reason.contains(ResolveReason::Date) && props.release_date.is_none(); + tmdb_missing || date_missing + } + #[allow(clippy::too_many_lines)] async fn process_task_static( input_name: &Arc, @@ -2037,7 +2659,7 @@ impl InputWorker { collector: &mut BatchResultCollector, db_handles: &mut HashMap, failed_clusters: &mut HashSet, - ) -> Result { + ) -> Result { let app_state = app_state_weak.and_then(Weak::upgrade).ok_or_else(|| shared::error::info_err!("AppState not available"))?; @@ -2155,7 +2777,30 @@ impl InputWorker { UpdateTask::ProbeLive { .. } => collector.live.len() > pre_live_updates, UpdateTask::ProbeStream { .. } => true, }; - Ok(task_changed) + let tmdb_pending = match task { + UpdateTask::ResolveVod { reason, .. } => { + if reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) { + collector + .vod + .get(pre_vod_updates) + .is_none_or(|(_, props)| Self::vod_tmdb_pending_for_reason(props, *reason)) + } else { + false + } + } + UpdateTask::ResolveSeries { reason, .. } => { + if reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) { + collector + .series + .get(pre_series_updates) + .is_none_or(|(_, props)| Self::series_tmdb_pending_for_reason(props, *reason)) + } else { + false + } + } + UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. } => false, + }; + Ok(ProcessTaskOutcome { task_changed, tmdb_pending }) } Err(e) => Err(e), } @@ -2430,6 +3075,54 @@ mod tests { } } + #[tokio::test] + async fn submit_task_identical_resolve_merge_keeps_generation() { + let (tx, mut rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; + + let initial = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Tmdb]), + delay: 10, + }; + MetadataUpdateManager::submit_task( + tx.clone(), + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + initial, + ) + .await; + + let identical_merge = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Tmdb]), + delay: 10, + }; + MetadataUpdateManager::submit_task( + tx, + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + identical_merge, + ) + .await; + + let first_signal = rx.try_recv().expect("first signal should be queued"); + assert_eq!(first_signal, TaskKey::Vod(42)); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) + )); + + let entry = pending_tasks.get(&TaskKey::Vod(42)).expect("pending entry should exist"); + assert_eq!(entry.generation.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn submit_task_probe_stream_merge_keeps_existing_payload_when_present() { let (tx, mut rx) = mpsc::channel::(8); @@ -2512,6 +3205,42 @@ mod tests { } } + #[tokio::test] + async fn submit_task_with_closed_sender_does_not_report_merged_for_existing_pending_entry() { + let (tx, rx) = mpsc::channel::(8); + drop(rx); + + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let key = TaskKey::Vod(42); + let existing_task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Info]), + delay: 10, + }; + pending_tasks.insert(key.clone(), PendingTask::new(existing_task)); + + let incoming_task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Probe]), + delay: 2, + }; + + let result = MetadataUpdateManager::submit_task( + tx, + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + MetadataUpdateRuntimeSettings::default().max_queue_size, + incoming_task, + ) + .await; + + assert_eq!(result, SubmitTaskResult::ChannelClosed); + assert!(!pending_tasks.contains_key(&key)); + assert_eq!(pending_task_count.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn finalize_processed_task_success_requeues_when_generation_changed() { let (tx, rx) = mpsc::channel::(8); @@ -2545,9 +3274,10 @@ mod tests { retry_states: HashMap::new(), resolve_exhausted: HashMap::new(), last_cycle_completed_at_ts: None, - probe_retry_state_path: None, - probe_retry_loaded: false, - probe_retry_load_retry_at_ts: None, + metadata_retry_state_path: None, + metadata_retry_loaded: false, + metadata_retry_load_retry_at_ts: None, + last_retry_state_prune_at_ts: None, scheduled_requeues: Arc::new(DashMap::new()), }; @@ -2587,9 +3317,10 @@ mod tests { retry_states: HashMap::new(), resolve_exhausted: HashMap::new(), last_cycle_completed_at_ts: None, - probe_retry_state_path: None, - probe_retry_loaded: false, - probe_retry_load_retry_at_ts: None, + metadata_retry_state_path: None, + metadata_retry_loaded: false, + metadata_retry_load_retry_at_ts: None, + last_retry_state_prune_at_ts: None, scheduled_requeues: Arc::new(DashMap::new()), }; @@ -2641,31 +3372,76 @@ mod tests { } #[test] - fn probe_retry_state_disk_roundtrip() { + fn strip_tmdb_reasons_returns_none_for_tmdb_only_resolve_task() { + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(100), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Tmdb, ResolveReason::Date]), + delay: 5, + }; + + assert!(InputWorker::strip_tmdb_reasons(&task).is_none()); + } + + #[test] + fn strip_tmdb_reasons_keeps_non_tmdb_reasons() { + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(5), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Tmdb, ResolveReason::Probe, ResolveReason::Info]), + delay: 1, + }; + + let stripped = InputWorker::strip_tmdb_reasons(&task).expect("task should keep non-tmdb reasons"); + match stripped { + UpdateTask::ResolveSeries { reason, .. } => { + assert!(!reason.contains(ResolveReason::Tmdb)); + assert!(!reason.contains(ResolveReason::Date)); + assert!(reason.contains(ResolveReason::Probe)); + assert!(reason.contains(ResolveReason::Info)); + } + other => panic!("unexpected task type after strip: {other:?}"), + } + } + + #[test] + fn metadata_retry_state_disk_roundtrip() { let dir = tempdir().expect("tempdir should be created"); - let path = dir.path().join("probe_retry_state.db"); + let path = dir.path().join("metadata_retry_state.db"); let key = TaskKey::Stream { scope: Arc::from("input_a"), id: Arc::from("stream_1"), }; - let state = RetryState { - attempts: 3, - next_allowed_at_ts: 1_700_000_000, - cooldown_until_ts: Some(1_700_086_400), - last_error: Some("probe timeout".to_string()), + let state = TaskRetryState { + resolve: None, + probe: Some(RetryState { + attempts: 3, + next_allowed_at_ts: 1_700_000_000, + cooldown_until_ts: Some(1_700_086_400), + last_error: Some("probe timeout".to_string()), + }), + tmdb: Some(RetryState { + attempts: 0, + next_allowed_at_ts: 1_700_172_800, + cooldown_until_ts: Some(1_700_172_800), + last_error: Some("tmdb no match".to_string()), + }), + updated_at_ts: 1_700_000_000, }; - persist_probe_retry_state_to_disk(&path, &key, Some(&state)) + persist_metadata_retry_state_to_disk(&path, &key, Some(&state)) .expect("state persistence should succeed"); - let loaded = load_probe_retry_states_from_disk(&path).expect("state load should succeed"); + let loaded = load_metadata_retry_states_from_disk(&path).expect("state load should succeed"); let loaded_state = loaded.get(&key).expect("probe key should be present"); - assert_eq!(loaded_state.attempts, 3); - assert_eq!(loaded_state.cooldown_until_ts, Some(1_700_086_400)); - assert_eq!(loaded_state.last_error.as_deref(), Some("probe timeout")); + let loaded_probe = loaded_state.probe.as_ref().expect("probe retry state should be present"); + assert_eq!(loaded_probe.attempts, 3); + assert_eq!(loaded_probe.cooldown_until_ts, Some(1_700_086_400)); + assert_eq!(loaded_probe.last_error.as_deref(), Some("probe timeout")); + let loaded_tmdb = loaded_state.tmdb.as_ref().expect("tmdb retry state should be present"); + assert_eq!(loaded_tmdb.cooldown_until_ts, Some(1_700_172_800)); + assert_eq!(loaded_tmdb.last_error.as_deref(), Some("tmdb no match")); - persist_probe_retry_state_to_disk(&path, &key, None).expect("state clear should succeed"); - let cleared = load_probe_retry_states_from_disk(&path).expect("state reload should succeed"); + persist_metadata_retry_state_to_disk(&path, &key, None).expect("state clear should succeed"); + let cleared = load_metadata_retry_states_from_disk(&path).expect("state reload should succeed"); assert!(!cleared.contains_key(&key)); } @@ -2688,4 +3464,38 @@ mod tests { assert!(InputWorker::is_transient_worker_error(TASK_ERR_NO_CONNECTION)); assert!(!InputWorker::is_transient_worker_error("permanent error")); } + + #[test] + fn permanent_not_found_error_matches_standalone_markers() { + assert!(InputWorker::is_permanent_not_found_error("HTTP 404 Not Found")); + assert!(InputWorker::is_permanent_not_found_error("probe failed: 404: stream unavailable")); + assert!(InputWorker::is_permanent_not_found_error("resource not found on provider")); + } + + #[test] + fn permanent_not_found_error_ignores_partial_markers() { + assert!(!InputWorker::is_permanent_not_found_error("error code 1404 while probing")); + assert!(!InputWorker::is_permanent_not_found_error("status404unexpected")); + assert!(!InputWorker::is_permanent_not_found_error("movie not foundry metadata mismatch")); + } + + #[test] + fn retry_domain_uses_probe_for_probe_only_resolve_tasks() { + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(7), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Probe]), + delay: 0, + }; + assert_eq!(InputWorker::retry_domain_for_task(&task), RetryDomain::Probe); + } + + #[test] + fn retry_domain_keeps_resolve_for_mixed_resolve_tasks() { + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(11), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Probe, ResolveReason::Info]), + delay: 0, + }; + assert_eq!(InputWorker::retry_domain_for_task(&task), RetryDomain::Resolve); + } } diff --git a/backend/src/api/scheduler.rs b/backend/src/api/scheduler.rs index c2d9937b9..b7bf3ef11 100644 --- a/backend/src/api/scheduler.rs +++ b/backend/src/api/scheduler.rs @@ -111,11 +111,20 @@ fn run_playlist_update(client: &reqwest::Client, app_state: &Arc, targ } fn run_library_scan(client: &reqwest::Client, app_state: &Arc) { - if let Some(lib_config) = app_state.app_config.config.load().library.as_ref() { + let config = app_state.app_config.config.load(); + if let Some(lib_config) = config.library.as_ref() { if lib_config.enabled { if let Some(permit) = app_state.update_guard.try_library() { let event_manager = Arc::clone(&app_state.event_manager); - spawn_library_scan(event_manager, lib_config.clone(), client.clone(), false, "Scheduled ", permit); + spawn_library_scan( + event_manager, + lib_config.clone(), + config.metadata_update.clone(), + client.clone(), + false, + "Scheduled ", + permit, + ); } } } diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index c2f8c8641..b54d70b63 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -4,7 +4,7 @@ use crate::library::metadata::{MediaMetadata, MetadataSource, MovieMetadata, Ser use crate::library::scanner::ScannedMediaFile; use crate::library::tmdb_client::TmdbClient; use crate::library::{MediaGroup, MetadataStorage}; -use crate::model::LibraryConfig; +use crate::model::{LibraryConfig, MetadataUpdateConfig}; use crate::ptt::{ptt_parse_title, PttMetadata}; // Metadata resolver that tries multiple sources to get video metadata @@ -15,28 +15,47 @@ pub struct MetadataResolver { impl MetadataResolver { // Creates a new metadata resolver from configuration - pub fn new(config: Option<&LibraryConfig>, client: reqwest::Client) -> Self { - let storage = match config { + pub fn new( + library_config: Option<&LibraryConfig>, + metadata_update_config: Option<&MetadataUpdateConfig>, + client: reqwest::Client, + ) -> Self { + let storage = match library_config { None => None, Some(c) => { let storage_path = std::path::PathBuf::from(&c.metadata.path); Some(MetadataStorage::new(storage_path)) } }; - Self::from_config(config, client, storage) + Self::from_config(library_config, metadata_update_config, client, storage) } - pub fn from_config(config: Option<&LibraryConfig>, client: reqwest::Client, storage: Option) -> Self { - let tmdb_client = config.filter(|c| c.metadata.tmdb.enabled) - .zip(storage) - .map(|(c, s)|{ - let api_key = c.metadata.tmdb.api_key.as_ref().map_or_else(|| TMDB_API_KEY.to_string(), ToString::to_string); - TmdbClient::new(api_key, c.metadata.tmdb.rate_limit_ms, client, s) - }); + pub fn from_config( + library_config: Option<&LibraryConfig>, + metadata_update_config: Option<&MetadataUpdateConfig>, + client: reqwest::Client, + storage: Option, + ) -> Self { + let tmdb_client = match metadata_update_config.filter(|config| config.tmdb.enabled) { + Some(config) => { + if let Some(tmdb_storage) = storage { + let api_key = config + .tmdb + .api_key + .as_ref() + .map_or_else(|| TMDB_API_KEY.to_string(), ToString::to_string); + Some(TmdbClient::new(api_key, config.tmdb.rate_limit_ms, client, tmdb_storage)) + } else { + warn!("TMDB is enabled but metadata storage is unavailable; TMDB resolver is disabled."); + None + } + } + None => None, + }; Self { tmdb_client, - fallback_to_filename: config.is_some_and(|c|c.metadata.fallback_to_filename), + fallback_to_filename: library_config.is_some_and(|c|c.metadata.fallback_to_filename), } } @@ -196,13 +215,15 @@ impl MetadataResolver { #[cfg(test)] mod tests { use super::*; - use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; + use crate::model::{ + LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, MetadataUpdateConfig, TmdbConfig, + }; use std::path::PathBuf; use std::time::Duration; use shared::utils::Internable; use crate::library::{MediaClassification, MediaClassifier}; - fn create_test_config(tmdb_enabled: bool, fallback_filename: bool) -> LibraryConfig { + fn create_test_library_config(fallback_filename: bool) -> LibraryConfig { LibraryConfig { enabled: true, scan_directories: vec![], @@ -214,17 +235,6 @@ mod tests { jellyfin: true, plex: true, }, - tmdb: LibraryTmdbConfig { - enabled: tmdb_enabled, - api_key: if tmdb_enabled { - Some("test_key".to_string()) - } else { - None - }, - rate_limit_ms: 250, - cache_duration_days: 0, - language: "en-US".to_string(), - }, fallback_to_filename: fallback_filename, formats: vec![], }, @@ -235,6 +245,24 @@ mod tests { } } + fn create_test_metadata_update_config(tmdb_enabled: bool) -> MetadataUpdateConfig { + MetadataUpdateConfig { + tmdb: TmdbConfig { + enabled: tmdb_enabled, + api_key: if tmdb_enabled { + Some("test_key".to_string()) + } else { + None + }, + rate_limit_ms: 250, + cache_duration_days: 0, + language: "en-US".to_string(), + ..TmdbConfig::default() + }, + ..MetadataUpdateConfig::default() + } + } + fn create_test_file(name: &str) -> ScannedMediaFile { ScannedMediaFile { path: PathBuf::from(format!("/test/{name}")), @@ -248,13 +276,19 @@ mod tests { #[tokio::test] async fn test_resolve_from_filename_movie() { - let config = create_test_config(false, true); + let library_config = create_test_library_config(true); + let metadata_update_config = create_test_metadata_update_config(false); let client = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(30)) .build() .unwrap_or_else(|_| reqwest::Client::new()); - let resolver = MetadataResolver::from_config(Some(&config), client, Some(MetadataStorage::new(PathBuf::from("/tmp")))); + let resolver = MetadataResolver::from_config( + Some(&library_config), + Some(&metadata_update_config), + client, + Some(MetadataStorage::new(PathBuf::from("/tmp"))), + ); let file = create_test_file("The.Matrix.1999.1080p.mkv"); let metadata = match MediaClassifier::classify(&file) { MediaClassification::Movie { metadata, .. } | MediaClassification::Series { metadata, .. } => metadata, @@ -275,13 +309,19 @@ mod tests { #[tokio::test] async fn test_resolve_from_title_stream() { - let config = create_test_config(false, true); + let library_config = create_test_library_config(true); + let metadata_update_config = create_test_metadata_update_config(false); let client = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(30)) .build() .unwrap_or_else(|_| reqwest::Client::new()); - let resolver = MetadataResolver::from_config(Some(&config), client, Some(MetadataStorage::new(PathBuf::from("/tmp")))); + let resolver = MetadataResolver::from_config( + Some(&library_config), + Some(&metadata_update_config), + client, + Some(MetadataStorage::new(PathBuf::from("/tmp"))), + ); let metadata = resolver.resolve_from_title("Inception.2010", None, true, true).await; assert!(metadata.is_some()); @@ -302,8 +342,14 @@ mod tests { .timeout(Duration::from_secs(30)) .build() .unwrap_or_else(|_| reqwest::Client::new()); - let config = create_test_config(false, false); - let resolver = MetadataResolver::from_config(Some(&config), client, Some(MetadataStorage::new(PathBuf::from("/tmp")))); + let library_config = create_test_library_config(false); + let metadata_update_config = create_test_metadata_update_config(false); + let resolver = MetadataResolver::from_config( + Some(&library_config), + Some(&metadata_update_config), + client, + Some(MetadataStorage::new(PathBuf::from("/tmp"))), + ); let file = create_test_file("343jfkjh4789dkjfh934z3.Movie.mkv"); let metadata = match MediaClassifier::classify(&file) { MediaClassification::Movie { metadata, .. } | MediaClassification::Series { metadata, .. } => metadata, diff --git a/backend/src/library/processor.rs b/backend/src/library/processor.rs index 1f9810058..41c96faa6 100644 --- a/backend/src/library/processor.rs +++ b/backend/src/library/processor.rs @@ -4,7 +4,7 @@ use crate::library::metadata_resolver::MetadataResolver; use crate::library::metadata_storage::MetadataStorage; use crate::library::scanner::LibraryScanner; use crate::library::{MediaGroup, MediaGrouper}; -use crate::model::{AppConfig, LibraryConfig}; +use crate::model::{AppConfig, LibraryConfig, MetadataUpdateConfig}; use log::{debug, error, info, warn}; use shared::model::{LibraryMetadataFormat, LibraryScanResult}; use std::collections::HashMap; @@ -34,15 +34,19 @@ impl LibraryProcessor { error!("Failed to create HTTP client for LibraryProcessor, skipping library scan. Please check your configuration."); return None; }; - app_config.config.load().library.as_ref().map(|lib_cfg| Self::new(lib_cfg.clone(), client)) + let config = app_config.config.load(); + config + .library + .as_ref() + .map(|lib_cfg| Self::new(lib_cfg.clone(), config.metadata_update.as_ref(), client)) } // Creates a new Library processor with the given configuration - pub fn new(config: LibraryConfig, client: reqwest::Client) -> Self { + pub fn new(config: LibraryConfig, metadata_update_config: Option<&MetadataUpdateConfig>, client: reqwest::Client) -> Self { let storage_path = std::path::PathBuf::from(&config.metadata.path); let scanner = LibraryScanner::new(config.clone()); let storage = MetadataStorage::new(storage_path); - let resolver = MetadataResolver::from_config(Some(&config), client, Some(storage.clone())); + let resolver = MetadataResolver::from_config(Some(&config), metadata_update_config, client, Some(storage.clone())); Self { config, diff --git a/backend/src/library/scanner.rs b/backend/src/library/scanner.rs index 4e58f4903..9704479c3 100644 --- a/backend/src/library/scanner.rs +++ b/backend/src/library/scanner.rs @@ -279,7 +279,7 @@ impl LibraryScanner { mod tests { use shared::utils::Internable; use super::*; - use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; + use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig}; fn create_test_config() -> LibraryConfig { LibraryConfig { @@ -297,13 +297,6 @@ mod tests { jellyfin: false, plex: false, }, - tmdb: LibraryTmdbConfig { - enabled: false, - api_key: Some(String::new()), - rate_limit_ms: 250, - cache_duration_days: 0, - language: String::new(), - }, fallback_to_filename: true, formats: vec![], }, diff --git a/backend/src/model/config/app.rs b/backend/src/model/config/app.rs index 284c9af71..641195112 100644 --- a/backend/src/model/config/app.rs +++ b/backend/src/model/config/app.rs @@ -454,7 +454,7 @@ impl AppConfig { pub async fn is_ffprobe_enabled(&self) -> bool { let ffprobe_enabled_in_config = { let config = self.config.load(); - config.metadata_update.as_ref().is_some_and(|metadata| metadata.ffprobe_enabled) + config.metadata_update.as_ref().is_some_and(|metadata| metadata.ffprobe.enabled) }; if !ffprobe_enabled_in_config { return false; diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index d6e1cd67e..23c9b29a9 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -17,7 +17,6 @@ pub struct LibraryScanDirectory { pub struct LibraryMetadataConfig { pub path: String, pub read_existing: LibraryMetadataReadConfig, - pub tmdb: LibraryTmdbConfig, pub fallback_to_filename: bool, pub formats: Vec, } @@ -29,15 +28,6 @@ pub struct LibraryMetadataReadConfig { pub plex: bool, } -#[derive(Debug, Clone)] -pub struct LibraryTmdbConfig { - pub enabled: bool, - pub api_key: Option, - pub rate_limit_ms: u64, - pub cache_duration_days: u32, - pub language: String, -} - #[derive(Debug, Clone)] pub struct LibraryPlaylistConfig { pub movie_category: Arc, @@ -124,13 +114,6 @@ impl From<&LibraryConfigDto> for LibraryConfig { jellyfin: dto.metadata.read_existing.jellyfin, plex: dto.metadata.read_existing.plex, }, - tmdb: LibraryTmdbConfig { - enabled: dto.metadata.tmdb.enabled, - api_key: dto.metadata.tmdb.api_key.clone(), - rate_limit_ms: dto.metadata.tmdb.rate_limit_ms, - cache_duration_days: dto.metadata.tmdb.cache_duration_days, - language: dto.metadata.tmdb.language.clone(), - }, fallback_to_filename: dto.metadata.fallback_to_filename, formats: dto.metadata.formats.clone(), }, diff --git a/backend/src/model/config/metadata_update.rs b/backend/src/model/config/metadata_update.rs index 5c4ebc24e..4902b98d0 100644 --- a/backend/src/model/config/metadata_update.rs +++ b/backend/src/model/config/metadata_update.rs @@ -1,5 +1,7 @@ use crate::model::macros; -use shared::model::MetadataUpdateConfigDto; +use shared::model::{ + FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, TmdbConfigDto, +}; use shared::utils::{ default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, @@ -7,54 +9,104 @@ use shared::utils::{ default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, default_metadata_queue_log_interval, - default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, - default_metadata_retry_delay, default_metadata_worker_idle_timeout, parse_duration_seconds, parse_size_base_2, + default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, + default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_tmdb_cache_duration_days, + default_tmdb_language, default_tmdb_rate_limit_ms, parse_duration_seconds, parse_size_base_2, }; #[derive(Debug, Clone)] pub struct MetadataUpdateConfig { - pub queue_log_interval: String, - pub queue_log_interval_secs: u64, - pub progress_log_interval: String, - pub progress_log_interval_secs: u64, - pub max_resolve_retry_backoff: String, - pub max_resolve_retry_backoff_secs: u64, - pub resolve_min_retry_base: String, - pub resolve_min_retry_base_secs: u64, - pub resolve_exhaustion_reset_gap: String, - pub resolve_exhaustion_reset_gap_secs: u64, - pub probe_cooldown: String, - pub probe_cooldown_secs: u64, + pub log: MetadataLogConfig, + pub resolve: ResolveConfig, + pub probe: ProbeConfig, + pub ffprobe: FfprobeConfig, + pub tmdb: TmdbConfig, pub retry_delay: String, pub retry_delay_secs: u64, - pub probe_retry_load_retry_delay: String, - pub probe_retry_load_retry_delay_secs: u64, pub worker_idle_timeout: String, pub worker_idle_timeout_secs: u64, - pub probe_retry_backoff_step_1: String, - pub probe_retry_backoff_step_1_secs: u64, - pub probe_retry_backoff_step_2: String, - pub probe_retry_backoff_step_2_secs: u64, - pub probe_retry_backoff_step_3: String, - pub probe_retry_backoff_step_3_secs: u64, - pub max_attempts_resolve: u8, - pub max_attempts_probe: u8, - pub backoff_jitter_percent: u8, pub max_queue_size: usize, - pub ffprobe_enabled: bool, - pub ffprobe_timeout: Option, - pub ffprobe_analyze_duration: String, - pub ffprobe_analyze_duration_micros: u64, - pub ffprobe_probe_size: String, - pub ffprobe_probe_size_bytes: u64, - pub ffprobe_live_analyze_duration: String, - pub ffprobe_live_analyze_duration_micros: u64, - pub ffprobe_live_probe_size: String, - pub ffprobe_live_probe_size_bytes: u64, +} + +#[derive(Debug, Clone)] +pub struct MetadataLogConfig { + pub queue_interval: String, + pub queue_interval_secs: u64, + pub progress_interval: String, + pub progress_interval_secs: u64, +} + +#[derive(Debug, Clone)] +pub struct ResolveConfig { + pub max_retry_backoff: String, + pub max_retry_backoff_secs: u64, + pub min_retry_base: String, + pub min_retry_base_secs: u64, + pub exhaustion_reset_gap: String, + pub exhaustion_reset_gap_secs: u64, + pub max_attempts: u8, +} + +#[derive(Debug, Clone)] +pub struct ProbeConfig { + pub cooldown: String, + pub cooldown_secs: u64, + pub retry_load_retry_delay: String, + pub retry_load_retry_delay_secs: u64, + pub retry_backoff_step_1: String, + pub retry_backoff_step_1_secs: u64, + pub retry_backoff_step_2: String, + pub retry_backoff_step_2_secs: u64, + pub retry_backoff_step_3: String, + pub retry_backoff_step_3_secs: u64, + pub max_attempts: u8, + pub backoff_jitter_percent: u8, +} + +#[derive(Debug, Clone)] +pub struct FfprobeConfig { + pub enabled: bool, + pub timeout: Option, + pub analyze_duration: String, + pub analyze_duration_micros: u64, + pub probe_size: String, + pub probe_size_bytes: u64, + pub live_analyze_duration: String, + pub live_analyze_duration_micros: u64, + pub live_probe_size: String, + pub live_probe_size_bytes: u64, +} + +#[derive(Debug, Clone)] +pub struct TmdbConfig { + pub enabled: bool, + pub api_key: Option, + pub rate_limit_ms: u64, + pub cache_duration_days: u32, + pub language: String, + pub cooldown: String, + pub cooldown_secs: u64, +} + +impl Default for TmdbConfig { + fn default() -> Self { + let cooldown = default_metadata_tmdb_cooldown(); + Self { + enabled: false, + api_key: None, + rate_limit_ms: default_tmdb_rate_limit_ms(), + cache_duration_days: default_tmdb_cache_duration_days(), + language: default_tmdb_language(), + cooldown_secs: parse_duration_or_default(&cooldown, &default_metadata_tmdb_cooldown(), false), + cooldown, + } + } } impl Default for MetadataUpdateConfig { - fn default() -> Self { Self::from(&MetadataUpdateConfigDto::default()) } + fn default() -> Self { + Self::from(&MetadataUpdateConfigDto::default()) + } } macros::from_impl!(MetadataUpdateConfig); @@ -73,103 +125,6 @@ fn parse_size_or_default(value: &str, default_value: &str) -> u64 { .unwrap_or(1) } -struct ParsedMetadataUpdateNumbers { - queue_log_interval_secs: u64, - progress_log_interval_secs: u64, - max_resolve_retry_backoff_secs: u64, - resolve_min_retry_base_secs: u64, - resolve_exhaustion_reset_gap_secs: u64, - probe_cooldown_secs: u64, - retry_delay_secs: u64, - probe_retry_load_retry_delay_secs: u64, - worker_idle_timeout_secs: u64, - probe_retry_backoff_step_1_secs: u64, - probe_retry_backoff_step_2_secs: u64, - probe_retry_backoff_step_3_secs: u64, - ffprobe_analyze_duration_micros: u64, - ffprobe_probe_size_bytes: u64, - ffprobe_live_analyze_duration_micros: u64, - ffprobe_live_probe_size_bytes: u64, -} - -fn parse_numeric_fields(cfg: &MetadataUpdateConfigDto) -> ParsedMetadataUpdateNumbers { - let queue_log_interval_secs = - parse_duration_or_default(&cfg.queue_log_interval, &default_metadata_queue_log_interval(), false); - let progress_log_interval_secs = - parse_duration_or_default(&cfg.progress_log_interval, &default_metadata_progress_log_interval(), false); - let max_resolve_retry_backoff_secs = parse_duration_or_default( - &cfg.max_resolve_retry_backoff, - &default_metadata_max_resolve_retry_backoff(), - false, - ); - let resolve_min_retry_base_secs = - parse_duration_or_default(&cfg.resolve_min_retry_base, &default_metadata_resolve_min_retry_base(), false); - let resolve_exhaustion_reset_gap_secs = parse_duration_or_default( - &cfg.resolve_exhaustion_reset_gap, - &default_metadata_resolve_exhaustion_reset_gap(), - false, - ); - let probe_cooldown_secs = parse_duration_or_default(&cfg.probe_cooldown, &default_metadata_probe_cooldown(), false); - let retry_delay_secs = parse_duration_or_default(&cfg.retry_delay, &default_metadata_retry_delay(), false); - let probe_retry_load_retry_delay_secs = parse_duration_or_default( - &cfg.probe_retry_load_retry_delay, - &default_metadata_probe_retry_load_retry_delay(), - false, - ); - let worker_idle_timeout_secs = - parse_duration_or_default(&cfg.worker_idle_timeout, &default_metadata_worker_idle_timeout(), false); - let probe_retry_backoff_step_1_secs = parse_duration_or_default( - &cfg.probe_retry_backoff_step_1, - &default_metadata_probe_retry_backoff_step_1(), - false, - ); - let probe_retry_backoff_step_2_secs = parse_duration_or_default( - &cfg.probe_retry_backoff_step_2, - &default_metadata_probe_retry_backoff_step_2(), - false, - ); - let probe_retry_backoff_step_3_secs = parse_duration_or_default( - &cfg.probe_retry_backoff_step_3, - &default_metadata_probe_retry_backoff_step_3(), - false, - ); - let ffprobe_analyze_duration_micros = parse_duration_or_default( - &cfg.ffprobe_analyze_duration, - &default_metadata_ffprobe_analyze_duration(), - true, - ) - .saturating_mul(1_000_000); - let ffprobe_probe_size_bytes = - parse_size_or_default(&cfg.ffprobe_probe_size, &default_metadata_ffprobe_probe_size()); - let ffprobe_live_analyze_duration_micros = parse_duration_or_default( - &cfg.ffprobe_live_analyze_duration, - &default_metadata_ffprobe_live_analyze_duration(), - true, - ) - .saturating_mul(1_000_000); - let ffprobe_live_probe_size_bytes = - parse_size_or_default(&cfg.ffprobe_live_probe_size, &default_metadata_ffprobe_live_probe_size()); - - ParsedMetadataUpdateNumbers { - queue_log_interval_secs, - progress_log_interval_secs, - max_resolve_retry_backoff_secs, - resolve_min_retry_base_secs, - resolve_exhaustion_reset_gap_secs, - probe_cooldown_secs, - retry_delay_secs, - probe_retry_load_retry_delay_secs, - worker_idle_timeout_secs, - probe_retry_backoff_step_1_secs, - probe_retry_backoff_step_2_secs, - probe_retry_backoff_step_3_secs, - ffprobe_analyze_duration_micros, - ffprobe_probe_size_bytes, - ffprobe_live_analyze_duration_micros, - ffprobe_live_probe_size_bytes, - } -} - impl From<&MetadataUpdateConfigDto> for MetadataUpdateConfig { fn from(dto: &MetadataUpdateConfigDto) -> Self { // ConfigDto::prepare() should already normalize/validate, but conversion stays defensive. @@ -178,47 +133,26 @@ impl From<&MetadataUpdateConfigDto> for MetadataUpdateConfig { normalized = MetadataUpdateConfigDto::default(); let _ = normalized.prepare(); } - let parsed = parse_numeric_fields(&normalized); Self { - queue_log_interval: normalized.queue_log_interval, - queue_log_interval_secs: parsed.queue_log_interval_secs, - progress_log_interval: normalized.progress_log_interval, - progress_log_interval_secs: parsed.progress_log_interval_secs, - max_resolve_retry_backoff: normalized.max_resolve_retry_backoff, - max_resolve_retry_backoff_secs: parsed.max_resolve_retry_backoff_secs, - resolve_min_retry_base: normalized.resolve_min_retry_base, - resolve_min_retry_base_secs: parsed.resolve_min_retry_base_secs, - resolve_exhaustion_reset_gap: normalized.resolve_exhaustion_reset_gap, - resolve_exhaustion_reset_gap_secs: parsed.resolve_exhaustion_reset_gap_secs, - probe_cooldown: normalized.probe_cooldown, - probe_cooldown_secs: parsed.probe_cooldown_secs, + log: MetadataLogConfig::from(&normalized.log), + resolve: ResolveConfig::from(&normalized.resolve), + probe: ProbeConfig::from(&normalized.probe), + ffprobe: FfprobeConfig::from(&normalized.ffprobe), + tmdb: TmdbConfig::from(&normalized.tmdb), + retry_delay_secs: parse_duration_or_default( + &normalized.retry_delay, + &default_metadata_retry_delay(), + false, + ), retry_delay: normalized.retry_delay, - retry_delay_secs: parsed.retry_delay_secs, - probe_retry_load_retry_delay: normalized.probe_retry_load_retry_delay, - probe_retry_load_retry_delay_secs: parsed.probe_retry_load_retry_delay_secs, + worker_idle_timeout_secs: parse_duration_or_default( + &normalized.worker_idle_timeout, + &default_metadata_worker_idle_timeout(), + false, + ), worker_idle_timeout: normalized.worker_idle_timeout, - worker_idle_timeout_secs: parsed.worker_idle_timeout_secs, - probe_retry_backoff_step_1: normalized.probe_retry_backoff_step_1, - probe_retry_backoff_step_1_secs: parsed.probe_retry_backoff_step_1_secs, - probe_retry_backoff_step_2: normalized.probe_retry_backoff_step_2, - probe_retry_backoff_step_2_secs: parsed.probe_retry_backoff_step_2_secs, - probe_retry_backoff_step_3: normalized.probe_retry_backoff_step_3, - probe_retry_backoff_step_3_secs: parsed.probe_retry_backoff_step_3_secs, - max_attempts_resolve: normalized.max_attempts_resolve, - max_attempts_probe: normalized.max_attempts_probe, - backoff_jitter_percent: normalized.backoff_jitter_percent, - max_queue_size: normalized.max_queue_size, - ffprobe_enabled: normalized.ffprobe_enabled, - ffprobe_timeout: normalized.ffprobe_timeout, - ffprobe_analyze_duration: normalized.ffprobe_analyze_duration, - ffprobe_analyze_duration_micros: parsed.ffprobe_analyze_duration_micros, - ffprobe_probe_size: normalized.ffprobe_probe_size, - ffprobe_probe_size_bytes: parsed.ffprobe_probe_size_bytes, - ffprobe_live_analyze_duration: normalized.ffprobe_live_analyze_duration, - ffprobe_live_analyze_duration_micros: parsed.ffprobe_live_analyze_duration_micros, - ffprobe_live_probe_size: normalized.ffprobe_live_probe_size, - ffprobe_live_probe_size_bytes: parsed.ffprobe_live_probe_size_bytes, + max_queue_size: normalized.max_queue_size.max(1), } } } @@ -226,28 +160,198 @@ impl From<&MetadataUpdateConfigDto> for MetadataUpdateConfig { impl From<&MetadataUpdateConfig> for MetadataUpdateConfigDto { fn from(instance: &MetadataUpdateConfig) -> Self { Self { - queue_log_interval: instance.queue_log_interval.clone(), - progress_log_interval: instance.progress_log_interval.clone(), - max_resolve_retry_backoff: instance.max_resolve_retry_backoff.clone(), - resolve_min_retry_base: instance.resolve_min_retry_base.clone(), - resolve_exhaustion_reset_gap: instance.resolve_exhaustion_reset_gap.clone(), - probe_cooldown: instance.probe_cooldown.clone(), + log: MetadataLogConfigDto::from(&instance.log), + resolve: ResolveConfigDto::from(&instance.resolve), + probe: ProbeConfigDto::from(&instance.probe), + ffprobe: FfprobeConfigDto::from(&instance.ffprobe), + tmdb: TmdbConfigDto::from(&instance.tmdb), retry_delay: instance.retry_delay.clone(), - probe_retry_load_retry_delay: instance.probe_retry_load_retry_delay.clone(), worker_idle_timeout: instance.worker_idle_timeout.clone(), - probe_retry_backoff_step_1: instance.probe_retry_backoff_step_1.clone(), - probe_retry_backoff_step_2: instance.probe_retry_backoff_step_2.clone(), - probe_retry_backoff_step_3: instance.probe_retry_backoff_step_3.clone(), - max_attempts_resolve: instance.max_attempts_resolve, - max_attempts_probe: instance.max_attempts_probe, - backoff_jitter_percent: instance.backoff_jitter_percent, max_queue_size: instance.max_queue_size, - ffprobe_enabled: instance.ffprobe_enabled, - ffprobe_timeout: instance.ffprobe_timeout, - ffprobe_analyze_duration: instance.ffprobe_analyze_duration.clone(), - ffprobe_probe_size: instance.ffprobe_probe_size.clone(), - ffprobe_live_analyze_duration: instance.ffprobe_live_analyze_duration.clone(), - ffprobe_live_probe_size: instance.ffprobe_live_probe_size.clone(), + } + } +} + +impl From<&MetadataLogConfigDto> for MetadataLogConfig { + fn from(dto: &MetadataLogConfigDto) -> Self { + Self { + queue_interval_secs: parse_duration_or_default( + &dto.queue_interval, + &default_metadata_queue_log_interval(), + false, + ), + queue_interval: dto.queue_interval.clone(), + progress_interval_secs: parse_duration_or_default( + &dto.progress_interval, + &default_metadata_progress_log_interval(), + false, + ), + progress_interval: dto.progress_interval.clone(), + } + } +} + +impl From<&MetadataLogConfig> for MetadataLogConfigDto { + fn from(instance: &MetadataLogConfig) -> Self { + Self { + queue_interval: instance.queue_interval.clone(), + progress_interval: instance.progress_interval.clone(), + } + } +} + +impl From<&ResolveConfigDto> for ResolveConfig { + fn from(dto: &ResolveConfigDto) -> Self { + Self { + max_retry_backoff_secs: parse_duration_or_default( + &dto.max_retry_backoff, + &default_metadata_max_resolve_retry_backoff(), + false, + ), + max_retry_backoff: dto.max_retry_backoff.clone(), + min_retry_base_secs: parse_duration_or_default( + &dto.min_retry_base, + &default_metadata_resolve_min_retry_base(), + false, + ), + min_retry_base: dto.min_retry_base.clone(), + exhaustion_reset_gap_secs: parse_duration_or_default( + &dto.exhaustion_reset_gap, + &default_metadata_resolve_exhaustion_reset_gap(), + false, + ), + exhaustion_reset_gap: dto.exhaustion_reset_gap.clone(), + max_attempts: dto.max_attempts.max(1), + } + } +} + +impl From<&ResolveConfig> for ResolveConfigDto { + fn from(instance: &ResolveConfig) -> Self { + Self { + max_retry_backoff: instance.max_retry_backoff.clone(), + min_retry_base: instance.min_retry_base.clone(), + exhaustion_reset_gap: instance.exhaustion_reset_gap.clone(), + max_attempts: instance.max_attempts, + } + } +} + +impl From<&ProbeConfigDto> for ProbeConfig { + fn from(dto: &ProbeConfigDto) -> Self { + Self { + cooldown_secs: parse_duration_or_default(&dto.cooldown, &default_metadata_probe_cooldown(), false), + cooldown: dto.cooldown.clone(), + retry_load_retry_delay_secs: parse_duration_or_default( + &dto.retry_load_retry_delay, + &default_metadata_probe_retry_load_retry_delay(), + false, + ), + retry_load_retry_delay: dto.retry_load_retry_delay.clone(), + retry_backoff_step_1_secs: parse_duration_or_default( + &dto.retry_backoff_step_1, + &default_metadata_probe_retry_backoff_step_1(), + false, + ), + retry_backoff_step_1: dto.retry_backoff_step_1.clone(), + retry_backoff_step_2_secs: parse_duration_or_default( + &dto.retry_backoff_step_2, + &default_metadata_probe_retry_backoff_step_2(), + false, + ), + retry_backoff_step_2: dto.retry_backoff_step_2.clone(), + retry_backoff_step_3_secs: parse_duration_or_default( + &dto.retry_backoff_step_3, + &default_metadata_probe_retry_backoff_step_3(), + false, + ), + retry_backoff_step_3: dto.retry_backoff_step_3.clone(), + max_attempts: dto.max_attempts.max(1), + backoff_jitter_percent: dto.backoff_jitter_percent.min(95), + } + } +} + +impl From<&ProbeConfig> for ProbeConfigDto { + fn from(instance: &ProbeConfig) -> Self { + Self { + cooldown: instance.cooldown.clone(), + retry_load_retry_delay: instance.retry_load_retry_delay.clone(), + retry_backoff_step_1: instance.retry_backoff_step_1.clone(), + retry_backoff_step_2: instance.retry_backoff_step_2.clone(), + retry_backoff_step_3: instance.retry_backoff_step_3.clone(), + max_attempts: instance.max_attempts, + backoff_jitter_percent: instance.backoff_jitter_percent, + } + } +} + +impl From<&FfprobeConfigDto> for FfprobeConfig { + fn from(dto: &FfprobeConfigDto) -> Self { + Self { + enabled: dto.enabled, + timeout: dto.timeout, + analyze_duration_micros: parse_duration_or_default( + &dto.analyze_duration, + &default_metadata_ffprobe_analyze_duration(), + true, + ) + .saturating_mul(1_000_000), + analyze_duration: dto.analyze_duration.clone(), + probe_size_bytes: parse_size_or_default(&dto.probe_size, &default_metadata_ffprobe_probe_size()), + probe_size: dto.probe_size.clone(), + live_analyze_duration_micros: parse_duration_or_default( + &dto.live_analyze_duration, + &default_metadata_ffprobe_live_analyze_duration(), + true, + ) + .saturating_mul(1_000_000), + live_analyze_duration: dto.live_analyze_duration.clone(), + live_probe_size_bytes: parse_size_or_default( + &dto.live_probe_size, + &default_metadata_ffprobe_live_probe_size(), + ), + live_probe_size: dto.live_probe_size.clone(), + } + } +} + +impl From<&FfprobeConfig> for FfprobeConfigDto { + fn from(instance: &FfprobeConfig) -> Self { + Self { + enabled: instance.enabled, + timeout: instance.timeout, + analyze_duration: instance.analyze_duration.clone(), + probe_size: instance.probe_size.clone(), + live_analyze_duration: instance.live_analyze_duration.clone(), + live_probe_size: instance.live_probe_size.clone(), + } + } +} + +impl From<&TmdbConfigDto> for TmdbConfig { + fn from(dto: &TmdbConfigDto) -> Self { + Self { + enabled: dto.enabled, + api_key: dto.api_key.clone(), + rate_limit_ms: dto.rate_limit_ms, + cache_duration_days: dto.cache_duration_days, + language: dto.language.clone(), + cooldown_secs: parse_duration_or_default(&dto.cooldown, &default_metadata_tmdb_cooldown(), false), + cooldown: dto.cooldown.clone(), + } + } +} + +impl From<&TmdbConfig> for TmdbConfigDto { + fn from(instance: &TmdbConfig) -> Self { + Self { + enabled: instance.enabled, + api_key: instance.api_key.clone(), + rate_limit_ms: instance.rate_limit_ms, + cache_duration_days: instance.cache_duration_days, + language: instance.language.clone(), + cooldown: instance.cooldown.clone(), } } } diff --git a/backend/src/processing/processor/stream_probe.rs b/backend/src/processing/processor/stream_probe.rs index 01407b3b0..662cde188 100644 --- a/backend/src/processing/processor/stream_probe.rs +++ b/backend/src/processing/processor/stream_probe.rs @@ -3,6 +3,7 @@ use crate::model::ConfigInput; use crate::model::{AppConfig}; use crate::repository::{get_input_m3u_playlist_file_path, get_input_storage_path, get_input_local_library_playlist_file_path, xtream_get_file_path, BPlusTreeUpdate}; use crate::utils::{debug_if_enabled, ffmpeg}; +use crate::utils::ffmpeg::{ProbeFailureKind, ProbeUrlOutcome}; use log::{info, warn}; use shared::error::TuliproxError; use shared::model::{EpisodeStreamProperties, InputType, PlaylistItemType, StreamProperties, VideoStreamDetailProperties, VideoStreamProperties, LiveStreamProperties, M3uPlaylistItem, XtreamCluster, XtreamPlaylistItem}; @@ -98,17 +99,17 @@ pub async fn update_generic_stream_metadata( let probe_url = stream_url.to_string(); let config = app_config.config.load(); let metadata_update = config.metadata_update.clone().unwrap_or_default(); - let ffprobe_timeout = metadata_update.ffprobe_timeout.unwrap_or(60); + let ffprobe_timeout = metadata_update.ffprobe.timeout.unwrap_or(60); let user_agent = config.default_user_agent.clone(); let (analyze_duration, probe_size) = if item_type.is_live() { ( - metadata_update.ffprobe_live_analyze_duration_micros, - metadata_update.ffprobe_live_probe_size_bytes, + metadata_update.ffprobe.live_analyze_duration_micros, + metadata_update.ffprobe.live_probe_size_bytes, ) } else { ( - metadata_update.ffprobe_analyze_duration_micros, - metadata_update.ffprobe_probe_size_bytes, + metadata_update.ffprobe.analyze_duration_micros, + metadata_update.ffprobe.probe_size_bytes, ) }; @@ -126,9 +127,16 @@ pub async fn update_generic_stream_metadata( active_provider.release_handle(&handle).await; } - let Some((_quality, raw_video, raw_audio)) = probe_data else { - warn!("Probe failed or timed out for generic stream: {unique_id}"); - return Ok(GenericProbeOutcome::ProbeFailed); + let (raw_video, raw_audio) = match probe_data { + ProbeUrlOutcome::Success(_quality, raw_video, raw_audio) => (raw_video, raw_audio), + ProbeUrlOutcome::Failed(ProbeFailureKind::NotFound) => { + warn!("Probe target not found (404) for generic stream: {unique_id}"); + return Err(shared::error::info_err!("Probe target returned 404 Not Found for stream {unique_id}")); + } + ProbeUrlOutcome::Failed(ProbeFailureKind::Other) => { + warn!("Probe failed or timed out for generic stream: {unique_id}"); + return Ok(GenericProbeOutcome::ProbeFailed); + } }; // Hold the async file lock while the blocking DB update runs in a blocking thread. diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index 6cd24189d..bb7579d7d 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -5,6 +5,7 @@ use crate::model::{AppConfig, ConfigInput, ConfigInputFlags}; use shared::model::{LiveStreamProperties, StreamProperties, XtreamCluster, XtreamPlaylistItem}; use crate::repository::{get_input_storage_path, persist_input_live_info, BPlusTreeQuery, xtream_get_file_path}; use crate::utils::{debug_if_enabled}; +use crate::utils::ffmpeg::{ProbeFailureKind, ProbeUrlOutcome}; use log::{debug, warn}; use crate::processing::parser::xtream::create_xtream_url; use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType}; @@ -117,10 +118,10 @@ pub async fn update_live_stream_metadata( ); let config = app_config.config.load(); let metadata_update = config.metadata_update.clone().unwrap_or_default(); - let ffprobe_timeout = metadata_update.ffprobe_timeout.unwrap_or(60); + let ffprobe_timeout = metadata_update.ffprobe.timeout.unwrap_or(60); let user_agent = config.default_user_agent.clone(); - let analyze_duration = metadata_update.ffprobe_live_analyze_duration_micros; - let probe_size = metadata_update.ffprobe_live_probe_size_bytes; + let analyze_duration = metadata_update.ffprobe.live_analyze_duration_micros; + let probe_size = metadata_update.ffprobe.live_probe_size_bytes; let display_id = stream_id_opt.map_or_else(|| "StringID".to_string(), |v| v.to_string()); debug!("Probing Live Stream ID {} for input {}", display_id, input.name); @@ -131,27 +132,37 @@ pub async fn update_live_stream_metadata( properties.last_probed_timestamp = Some(now); let mut success = false; - if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url( + let mut not_found = false; + match crate::utils::ffmpeg::probe_url( &stream_url, user_agent.as_deref(), analyze_duration, probe_size, ffprobe_timeout, - ).await { - // 3. Update properties on success - if let Some(v) = raw_video { - properties.video = Some(v.to_string().into()); + ) + .await + { + ProbeUrlOutcome::Success(_quality, raw_video, raw_audio) => { + // 3. Update properties on success + if let Some(v) = raw_video { + properties.video = Some(v.to_string().into()); + } + if let Some(a) = raw_audio { + properties.audio = Some(a.to_string().into()); + } + properties.last_success_timestamp = Some(now); + success = true; + + debug_if_enabled!("Successfully probed Live Stream ID {}", display_id); } - if let Some(a) = raw_audio { - properties.audio = Some(a.to_string().into()); + ProbeUrlOutcome::Failed(ProbeFailureKind::NotFound) => { + warn!("Live stream probe target returned 404 for ID {} (Input: {})", display_id, input.name); + not_found = true; + } + ProbeUrlOutcome::Failed(ProbeFailureKind::Other) => { + warn!("Probe failed for Live Stream ID {} (Input: {})", display_id, input.name); + // We still persist the updated last_probed_timestamp so we don't retry immediately } - properties.last_success_timestamp = Some(now); - success = true; - - debug_if_enabled!("Successfully probed Live Stream ID {}", display_id); - } else { - warn!("Probe failed for Live Stream ID {} (Input: {})", display_id, input.name); - // We still persist the updated last_probed_timestamp so we don't retry immediately } // 4. Persist @@ -164,6 +175,9 @@ pub async fn update_live_stream_metadata( } if !success { + if not_found { + return Err(shared::error::info_err!("Probe failed with 404 Not Found for stream {display_id}")); + } // Return error to propagate failure up to task manager/logs return Err(shared::error::info_err!("Probe failed for stream {display_id}")); } diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index cf80e84be..aa361c3ae 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -1,6 +1,6 @@ use crate::api::model::UpdateTask; use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType, ResolveReason, ResolveReasonSet}; -use crate::library::MetadataResolver; +use crate::library::{MetadataResolver, MetadataStorage}; use crate::model::FetchedPlaylist; use crate::model::{AppConfig, ConfigTarget, MetadataUpdateConfig}; use crate::model::{ConfigInput, ConfigInputFlags, InputSource}; @@ -18,6 +18,7 @@ use crate::repository::{ get_input_storage_path, persist_input_series_info_batch, MemoryPlaylistSource, PlaylistSource, }; use crate::repository::{xtream_get_file_path, BPlusTreeQuery}; +use crate::utils::ffmpeg::{ProbeFailureKind, ProbeUrlOutcome}; use crate::utils::{debug_if_enabled, xtream}; use log::{debug, error, info, log_enabled, trace, warn, Level}; use parking_lot::Mutex; @@ -45,11 +46,11 @@ impl SeriesProbeSettings { pub fn from_metadata_update(metadata_update: Option<&MetadataUpdateConfig>) -> Self { let defaults = MetadataUpdateConfig::default(); Self { - timeout_secs: metadata_update.and_then(|cfg| cfg.ffprobe_timeout).unwrap_or(60), + timeout_secs: metadata_update.and_then(|cfg| cfg.ffprobe.timeout).unwrap_or(60), analyze_duration_micros: metadata_update - .map_or(defaults.ffprobe_analyze_duration_micros, |cfg| cfg.ffprobe_analyze_duration_micros), + .map_or(defaults.ffprobe.analyze_duration_micros, |cfg| cfg.ffprobe.analyze_duration_micros), probe_size_bytes: metadata_update - .map_or(defaults.ffprobe_probe_size_bytes, |cfg| cfg.ffprobe_probe_size_bytes), + .map_or(defaults.ffprobe.probe_size_bytes, |cfg| cfg.ffprobe.probe_size_bytes), } } } @@ -695,6 +696,7 @@ pub async fn update_series_metadata( let mut fetched_new = false; let mut properties_updated = false; + let mut probe_failure: Option = None; let display_id = series_id_opt.map_or_else(|| "StringID".to_string(), |v| v.to_string()); @@ -759,7 +761,12 @@ pub async fn update_series_metadata( { let config = app_config.config.load(); let library_config = config.library.as_ref(); - let meta_resolver = MetadataResolver::new(library_config, client.clone()); + let metadata_update_config = config.metadata_update.as_ref(); + let tmdb_storage = metadata_update_config + .filter(|cfg| cfg.tmdb.enabled) + .map(|_| MetadataStorage::new(storage_path.clone())); + let meta_resolver = + MetadataResolver::from_config(library_config, metadata_update_config, client.clone(), tmdb_storage); let mut meta = None; let mut tried_title = false; @@ -856,7 +863,7 @@ pub async fn update_series_metadata( ep.title, ep.season, ep.episode_num, missing_reason ); - if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url( + match crate::utils::ffmpeg::probe_url( &episode_url, user_agent.as_deref(), probe_settings.analyze_duration_micros, @@ -865,15 +872,25 @@ pub async fn update_series_metadata( ) .await { - if let Some(v) = raw_video { - ep.video = Some(v.to_string().into()); - properties_updated = true; + ProbeUrlOutcome::Success(_quality, raw_video, raw_audio) => { + if let Some(v) = raw_video { + ep.video = Some(v.to_string().into()); + properties_updated = true; + } + if let Some(a) = raw_audio { + ep.audio = Some(a.to_string().into()); + properties_updated = true; + } + probed_count += 1; } - if let Some(a) = raw_audio { - ep.audio = Some(a.to_string().into()); - properties_updated = true; + ProbeUrlOutcome::Failed(ProbeFailureKind::NotFound) => { + probe_failure = Some(ProbeFailureKind::NotFound); + } + ProbeUrlOutcome::Failed(ProbeFailureKind::Other) => { + if probe_failure.is_none() { + probe_failure = Some(ProbeFailureKind::Other); + } } - probed_count += 1; } if let Some(h) = temp_handle { @@ -897,6 +914,19 @@ pub async fn update_series_metadata( } } + let probe_only_unresolved = do_probe && !fetch_info && !resolve_tmdb && !properties_updated && !fetched_new; + if probe_only_unresolved { + if let Some(kind) = probe_failure { + let err = match kind { + ProbeFailureKind::NotFound => { + shared::error::info_err!("Probe failed with 404 Not Found for Series {display_id}") + } + ProbeFailureKind::Other => shared::error::info_err!("Probe failed for Series {display_id}"), + }; + return Err(err); + } + } + // 4. Persist if properties_updated || fetched_new { if save { diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index b708c3ea1..c485bd427 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -1,6 +1,6 @@ use crate::api::model::{ActiveProviderManager, ProviderHandle}; use crate::api::model::{ProviderIdType, ResolveReason, ResolveReasonSet, UpdateTask}; -use crate::library::MetadataResolver; +use crate::library::{MetadataResolver, MetadataStorage}; use crate::model::FetchedPlaylist; use crate::model::InputSource; use crate::model::{AppConfig, ConfigTarget}; @@ -16,6 +16,7 @@ use crate::repository::get_input_storage_path; use crate::repository::persist_input_vod_info; use crate::repository::persist_input_vod_info_batch; use crate::repository::{xtream_get_file_path, BPlusTreeQuery}; +use crate::utils::ffmpeg::{ProbeFailureKind, ProbeUrlOutcome}; use crate::utils::{debug_if_enabled, trace_if_enabled, xtream}; use log::{debug, error, info, log_enabled, trace, warn, Level}; use parking_lot::Mutex; @@ -577,6 +578,7 @@ pub async fn update_vod_metadata( let mut fetched_new = false; let mut properties_updated = false; + let mut probe_failure: Option = None; // Determine the title to use for logging and fallback. // Cloning here to satisfy borrow checker when 'props' is mutated later. @@ -676,7 +678,12 @@ pub async fn update_vod_metadata( if missing_tmdb || still_missing_date { let config = app_config.config.load(); let library_config = config.library.as_ref(); - let meta_resolver = MetadataResolver::new(library_config, client.clone()); + let metadata_update_config = config.metadata_update.as_ref(); + let tmdb_storage = metadata_update_config + .filter(|cfg| cfg.tmdb.enabled) + .map(|_| MetadataStorage::new(storage_path.clone())); + let meta_resolver = + MetadataResolver::from_config(library_config, metadata_update_config, client.clone(), tmdb_storage); let mut meta = None; let mut tried_title = false; @@ -767,10 +774,10 @@ pub async fn update_vod_metadata( let config = app_config.config.load(); let metadata_update = config.metadata_update.clone().unwrap_or_default(); - let ffprobe_timeout = metadata_update.ffprobe_timeout.unwrap_or(60); + let ffprobe_timeout = metadata_update.ffprobe.timeout.unwrap_or(60); let user_agent = config.default_user_agent.clone(); - let analyze_duration = metadata_update.ffprobe_analyze_duration_micros; - let probe_size = metadata_update.ffprobe_probe_size_bytes; + let analyze_duration = metadata_update.ffprobe.analyze_duration_micros; + let probe_size = metadata_update.ffprobe.probe_size_bytes; // Acquire Connection logic let temp_handle = if active_handle.is_some() { @@ -779,28 +786,38 @@ pub async fn update_vod_metadata( active_provider.acquire_connection_for_probe(&input.name).await }; - if active_handle.is_some() || temp_handle.is_some() { - debug_if_enabled!("Probing VOD '{}' (ID: {})", display_title, display_id); - if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url( - &stream_url, - user_agent.as_deref(), - analyze_duration, - probe_size, - ffprobe_timeout, - ) - .await - { - if let Some(details) = properties.details.as_mut() { - if let Some(v) = raw_video { - details.video = Some(v.to_string().into()); - properties_updated = true; + if active_handle.is_some() || temp_handle.is_some() { + debug_if_enabled!("Probing VOD '{}' (ID: {})", display_title, display_id); + match crate::utils::ffmpeg::probe_url( + &stream_url, + user_agent.as_deref(), + analyze_duration, + probe_size, + ffprobe_timeout, + ) + .await + { + ProbeUrlOutcome::Success(_quality, raw_video, raw_audio) => { + if let Some(details) = properties.details.as_mut() { + if let Some(v) = raw_video { + details.video = Some(v.to_string().into()); + properties_updated = true; + } + if let Some(a) = raw_audio { + details.audio = Some(a.to_string().into()); + properties_updated = true; + } + } } - if let Some(a) = raw_audio { - details.audio = Some(a.to_string().into()); - properties_updated = true; + ProbeUrlOutcome::Failed(ProbeFailureKind::NotFound) => { + probe_failure = Some(ProbeFailureKind::NotFound); + } + ProbeUrlOutcome::Failed(ProbeFailureKind::Other) => { + if probe_failure.is_none() { + probe_failure = Some(ProbeFailureKind::Other); + } } } - } if let Some(h) = temp_handle { active_provider.release_handle(&h).await; } @@ -810,6 +827,19 @@ pub async fn update_vod_metadata( } } + let probe_only_unresolved = do_probe && !fetch_info && !resolve_tmdb && !properties_updated && !fetched_new; + if probe_only_unresolved { + if let Some(kind) = probe_failure { + let err = match kind { + ProbeFailureKind::NotFound => { + shared::error::info_err!("Probe failed with 404 Not Found for VOD {display_id}") + } + ProbeFailureKind::Other => shared::error::info_err!("Probe failed for VOD {display_id}"), + }; + return Err(err); + } + } + // 4. Persist if updated if properties_updated || fetched_new { if save { diff --git a/backend/src/utils/ffmpeg.rs b/backend/src/utils/ffmpeg.rs index af5c71f6e..88eb6cf6e 100644 --- a/backend/src/utils/ffmpeg.rs +++ b/backend/src/utils/ffmpeg.rs @@ -5,6 +5,17 @@ use serde_json::Value; use shared::model::MediaQuality; use shared::utils::sanitize_sensitive_info; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProbeFailureKind { + NotFound, + Other, +} + +pub enum ProbeUrlOutcome { + Success(MediaQuality, Option, Option), + Failed(ProbeFailureKind), +} + // Checks if ffprobe is available in the system path pub async fn check_ffprobe_availability() -> bool { match Command::new("ffprobe").arg("-version").output().await { @@ -13,7 +24,18 @@ pub async fn check_ffprobe_availability() -> bool { } } -pub async fn probe_url(url: &str, user_agent: Option<&str>, analyze_duration: u64, probe_size: u64, timeout_secs: u64) -> Option<(MediaQuality, Option, Option)> { +fn is_not_found_probe_error(stderr: &str) -> bool { + let normalized = stderr.to_ascii_lowercase(); + normalized.contains("404") || normalized.contains("not found") +} + +pub async fn probe_url( + url: &str, + user_agent: Option<&str>, + analyze_duration: u64, + probe_size: u64, + timeout_secs: u64, +) -> ProbeUrlOutcome { // Determine timeout: Ensure it's at least as long as the analyze duration + buffer, // but respect the user setting if it's longer. let analyze_overhead = Duration::from_micros(analyze_duration) + Duration::from_secs(5); @@ -46,7 +68,10 @@ pub async fn probe_url(url: &str, user_agent: Option<&str>, analyze_duration: u6 if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); debug!("ffprobe failed for {}: {}", sanitize_sensitive_info(url), sanitize_sensitive_info(&stderr)); - return None; + if is_not_found_probe_error(&stderr) { + return ProbeUrlOutcome::Failed(ProbeFailureKind::NotFound); + } + return ProbeUrlOutcome::Failed(ProbeFailureKind::Other); } let stdout = String::from_utf8_lossy(&output.stdout); @@ -94,7 +119,7 @@ pub async fn probe_url(url: &str, user_agent: Option<&str>, analyze_duration: u6 if video_info.is_some() || audio_info.is_some() { let mq = MediaQuality::from_ffprobe_info(audio_info.as_deref(), video_info.as_deref()); if let Some(quality) = mq { - return Some((quality, raw_video_json, raw_audio_json)); + return ProbeUrlOutcome::Success(quality, raw_video_json, raw_audio_json); } } } @@ -110,5 +135,5 @@ pub async fn probe_url(url: &str, user_agent: Option<&str>, analyze_duration: u6 } } - None -} \ No newline at end of file + ProbeUrlOutcome::Failed(ProbeFailureKind::Other) +} diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 376429350..02242c0fe 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -242,6 +242,7 @@ "METADATA_RESOLVE_MIN_RETRY_BASE": "Resolve min retry base", "METADATA_RESOLVE_EXHAUSTION_RESET_GAP": "Resolve exhaustion reset gap", "METADATA_PROBE_COOLDOWN": "Probe cooldown", + "METADATA_TMDB_COOLDOWN": "TMDB cooldown", "METADATA_RETRY_DELAY": "Retry delay", "METADATA_PROBE_RETRY_LOAD_RETRY_DELAY": "Probe state reload retry delay", "METADATA_WORKER_IDLE_TIMEOUT": "Worker idle timeout", @@ -452,6 +453,7 @@ "API_PROXY_CONFIG": "Api Proxy Config", "USE_USER_DB": "Use User DB", "ADD_SERVER": "Add Server", + "FFPROBE": "FFprobe", "FFPROBE_ENABLED": "FFprobe Enabled", "FFPROBE_TIMEOUT": "FFprobe Timeout (sec)", "FAILOVER_REDIRECT_PATTERNS": "Failover Redirect Patterns", @@ -651,6 +653,9 @@ }, "EXPLANATION": { "DEFAULT": "No dedicated explanation is available for this field yet.", + "RESOLVE": "Resolve controls metadata lookup retries.\n\nUse this group to tune how often failed metadata lookups are retried and how long the system should wait between retries.", + "PROBE": "Probe controls technical stream analysis retries.\n\nUse this group to define retry steps, retry limits, and cooldown behavior for FFprobe-based stream checks.", + "FFPROBE": "FFprobe controls how stream analysis is performed.\n\nUse this group to configure timeout, analyze duration, and probe size for both VOD/Series and Live streams.", "API_PROXY_CONFIG": { "SERVER": "Configuration value for server.", "USE_USER_DB": "If enabled, API proxy users are loaded from the user database." @@ -803,13 +808,6 @@ "MOVIE_CATEGORY": "Configuration value for movie category.", "SERIES_CATEGORY": "Configuration value for series category." }, - "LIBRARY_TMDB_CONFIG": { - "API_KEY": "Configuration value for api key.", - "CACHE_DURATION_DAYS": "Configuration value for cache duration days.", - "ENABLED": "Configuration value for enabled.", - "LANGUAGE": "Configuration value for language.", - "RATE_LIMIT_MS": "Configuration value for rate limit ms." - }, "LOG_CONFIG": { "LOG_ACTIVE_USER": "Configuration value for log active user.", "LOG_LEVEL": "Configuration value for log level.", @@ -839,29 +837,45 @@ "WORKING_DIR": "Base working directory used by tuliprox." }, "METADATA_UPDATE_CONFIG": { - "FFPROBE_ENABLED": "Enables or disables technical stream probing with ffprobe.\n\nWhen disabled, probe tasks are considered complete without reading the stream itself. Keep this disabled if you want minimum network load, and enable it when you need codec/resolution/audio metadata to be filled automatically.", - "FFPROBE_TIMEOUT": "Maximum runtime of one ffprobe call in seconds.\n\nIf a stream does not respond before this timeout, the probe attempt is treated as failed and follows retry rules.\n\nLeave empty to use the built-in default timeout (60 seconds).", - "FFPROBE_ANALYZE_DURATION": "How much media time ffprobe may analyze for VOD/Series/Episodes probes.\n\nHigher values can improve metadata detection on difficult files, but increase probe time and provider traffic.\n\nUse duration units like `5s`, `30s`, `1m`, `1h`.", - "FFPROBE_PROBE_SIZE": "How much data ffprobe may read for VOD/Series/Episodes probes.\n\nHigher values can improve detection reliability, especially on streams with delayed headers, but use more bandwidth.\n\nUse size units like `512KB`, `5MB`, `10MB`.", - "FFPROBE_LIVE_ANALYZE_DURATION": "How much media time ffprobe may analyze for Live probes.\n\nLive streams should usually use smaller values than VOD to keep probing fast and reduce waiting time.\n\nUse duration units like `2s`, `5s`, `10s`.", - "FFPROBE_LIVE_PROBE_SIZE": "How much data ffprobe may read for Live probes.\n\nSmaller values reduce load and speed up checks; larger values can help on unstable or delayed live streams.\n\nUse size units like `512KB`, `1MB`, `5MB`.", - "MAX_ATTEMPTS_RESOLVE": "Maximum number of resolve attempts per item inside one update cycle.\n\nAfter this limit is reached, the item is not retried again in the same cycle. A later cycle can start over after `resolve_exhaustion_reset_gap` has passed.", - "MAX_ATTEMPTS_PROBE": "Maximum number of probe attempts before an item enters cooldown.\n\nAfter the limit is reached, probing is paused for the configured `probe_cooldown`, then starts again from attempt 1.", - "BACKOFF_JITTER_PERCENT": "Adds random spread to retry delays to avoid many tasks retrying at exactly the same second.\n\nExample: with 20% jitter, retries are slightly shifted around the base delay. This reduces retry spikes under load.", - "RESOLVE_MIN_RETRY_BASE": "Minimum base delay used for resolve retries.\n\nEven if a task asks for an immediate retry, this minimum is enforced to prevent tight retry loops.\n\nUse duration units like `5s`, `30s`, `1m`.", - "MAX_RESOLVE_RETRY_BACKOFF": "Upper limit for resolve retry delay.\n\nExponential backoff grows on repeated failures, but never exceeds this value.\n\nUse duration units like `1m`, `10m`, `1h`.", - "PROBE_RETRY_BACKOFF_STEP_1": "Delay before the next probe after the first probe failure.\n\nThis is the first step of probe backoff.\n\nUse duration units like `10m`.", - "PROBE_RETRY_BACKOFF_STEP_2": "Delay before the next probe after the second probe failure.\n\nThis is the second step of probe backoff and is usually longer than step 1.\n\nUse duration units like `30m`.", - "PROBE_RETRY_BACKOFF_STEP_3": "Delay before the next probe after the third and later probe failures (until max attempts is reached).\n\nThis is the largest regular probe backoff step before cooldown.\n\nUse duration units like `1h`.", "RETRY_DELAY": "Minimum delay for transient requeue scheduling.\n\nUse this to prevent immediate hot-loop retries when tasks are temporarily deferred.\n\nUse duration units like `2s`, `5s`, `10s`.", - "PROBE_RETRY_LOAD_RETRY_DELAY": "Delay before retrying to load persisted probe retry state after a load failure.\n\nThis controls how quickly the worker retries reading retry-state files if disk access failed previously.\n\nUse duration units like `30s`, `1m`, `5m`.", - "RESOLVE_EXHAUSTION_RESET_GAP": "Minimum time gap required before resolve exhaustion is reset for a new update cycle.\n\nThis avoids permanently failing items being retried too aggressively across frequent updates.\n\nUse duration units like `30m`, `1h`, `2h`.", - "PROBE_COOLDOWN": "Cooldown period after probe attempts are exhausted.\n\nDuring cooldown, the item is quarantined and not probed again. After cooldown ends, probe retries start from attempt 1 again.\n\nUse duration units like `1d`, `3d`, `7d`.", "MAX_QUEUE_SIZE": "Maximum number of metadata tasks kept in memory for one input worker.\n\nWhen this limit is reached, additional tasks are rejected to protect system stability and memory usage.", - "QUEUE_LOG_INTERVAL": "How often queue-size status is logged.\n\nLower values produce more frequent operational logs; higher values reduce log noise.\n\nUse duration units like `15s`, `30s`, `1m`.", - "PROGRESS_LOG_INTERVAL": "How often progress logs are emitted while tasks are actively processed.\n\nLower values give more live visibility; higher values reduce log volume.\n\nUse duration units like `10s`, `30s`, `1m`.", "WORKER_IDLE_TIMEOUT": "How long a metadata worker waits for new tasks while idle before it can stop itself.\n\nA lower value releases resources sooner; a higher value keeps workers warm for bursty workloads.\n\nUse duration units like `30s`, `1m`, `5m`." }, + "METADATA_LOG_CONFIG": { + "QUEUE_INTERVAL": "How often the app writes queue status messages to the log.\n\nUse a smaller value if you want to see activity more often. Use a larger value if logs feel too noisy.\n\nExamples: `15s`, `30s`, `1m`.", + "PROGRESS_INTERVAL": "How often the app writes progress updates while metadata tasks are running.\n\nShort intervals show more live progress, long intervals keep logs cleaner.\n\nExamples: `10s`, `30s`, `1m`." + }, + "RESOLVE_CONFIG": { + "MAX_RETRY_BACKOFF": "Maximum waiting time between repeated resolve retries.\n\nIf something keeps failing, the app waits longer each time, but never longer than this value.\n\nExamples: `1m`, `10m`, `1h`.", + "MIN_RETRY_BASE": "Minimum waiting time before a resolve retry.\n\nThis prevents very fast retry loops when a source has temporary issues.\n\nExamples: `5s`, `30s`, `1m`.", + "MAX_ATTEMPTS": "How many resolve tries are allowed for one item in one update cycle.\n\nAfter this limit, the item is paused for now and retried in a later cycle.", + "EXHAUSTION_RESET_GAP": "How long the app waits before allowing a fresh resolve cycle for previously exhausted items.\n\nUse higher values to avoid repeatedly hammering the same failing entries.\n\nExamples: `30m`, `1h`, `2h`." + }, + "PROBE_CONFIG": { + "COOLDOWN": "Pause time after probe retries are exhausted.\n\nDuring cooldown, that item is skipped for probing. After cooldown ends, probing can start again.\n\nExamples: `1d`, `3d`, `7d`.", + "RETRY_LOAD_RETRY_DELAY": "Wait time before retrying to load saved retry-state data after a read failure.\n\nThis helps when storage is temporarily busy or unavailable.\n\nExamples: `30s`, `1m`, `5m`.", + "RETRY_BACKOFF_STEP_1": "Wait time after the first probe failure.\n\nUse this as a gentle first retry step.\n\nExample: `10m`.", + "RETRY_BACKOFF_STEP_2": "Wait time after the second probe failure.\n\nUsually set higher than step 1.\n\nExample: `30m`.", + "RETRY_BACKOFF_STEP_3": "Wait time after the third and later probe failures.\n\nThis is the longest regular probe delay before cooldown takes over.\n\nExample: `1h`.", + "MAX_ATTEMPTS": "How many probe tries are allowed before cooldown starts.\n\nLower values reduce provider load, higher values can recover more transient failures.", + "BACKOFF_JITTER_PERCENT": "Adds small randomness to retry timing.\n\nThis prevents many retries from happening at exactly the same second.\n\nTypical value: `20`." + }, + "FFPROBE_CONFIG": { + "ENABLED": "Turns FFprobe stream analysis on or off.\n\nWhen off, the app skips technical probing (video/audio details).", + "TIMEOUT": "Maximum seconds one FFprobe run may take.\n\nIf the stream is too slow, probing stops when this limit is reached.\n\nLeave empty to use the default (60 seconds).", + "ANALYZE_DURATION": "How much playback time FFprobe is allowed to inspect for VOD/Series.\n\nHigher values can find more details but take longer.\n\nExamples: `10s`, `30s`, `1m`.", + "PROBE_SIZE": "How much data FFprobe may read for VOD/Series.\n\nHigher values can improve detection but use more bandwidth.\n\nExamples: `1MB`, `5MB`, `10MB`.", + "LIVE_ANALYZE_DURATION": "How much playback time FFprobe may inspect for live streams.\n\nLive checks are usually faster with small values.\n\nExamples: `2s`, `5s`, `10s`.", + "LIVE_PROBE_SIZE": "How much data FFprobe may read for live streams.\n\nSmaller values reduce load and make checks faster.\n\nExamples: `512KB`, `1MB`, `5MB`." + }, + "TMDB_CONFIG": { + "ENABLED": "Enables metadata lookup from TMDB.\n\nTurn this off if you do not want TMDB-based title matching.", + "API_KEY": "Your TMDB API key.\n\nYou only need this if you want to use your own TMDB account limits.", + "RATE_LIMIT_MS": "Minimum wait time between TMDB requests in milliseconds.\n\nHigher values are gentler to the TMDB API.\n\nExamples: `250`, `500`, `1000`.", + "CACHE_DURATION_DAYS": "How many days TMDB results stay cached before refresh.\n\nLonger cache means fewer API calls.", + "LANGUAGE": "Preferred language for TMDB titles and metadata.\n\nExample: `en-US`.", + "COOLDOWN": "Pause after a successful TMDB lookup that returned no match.\n\nThis prevents endless re-tries for titles that currently do not exist in TMDB.\n\nExamples: `1d`, `3d`, `7d`." + }, "MESSAGING_CONFIG": { "NOTIFY_ON": "Configuration value for notify on." }, diff --git a/frontend/scss/app/components/_form.scss b/frontend/scss/app/components/_form.scss index d76412b29..3f29999b8 100644 --- a/frontend/scss/app/components/_form.scss +++ b/frontend/scss/app/components/_form.scss @@ -86,16 +86,19 @@ .tp__field-explanation-dialog { display: flex; flex-flow: column; - gap: var(--gap-default); + gap: var(--gap-large); min-width: min(32rem, 88vw); - h2 { - margin: 0; + &__header { + h2 { + color: var(--modest-text-color); + margin: 0; + } } - p { - margin: 0; - white-space: pre-wrap; - color: var(--modest-text-color); + &__body { + p { + white-space: pre-wrap; + } } } diff --git a/frontend/src/app/components/config/config_update.rs b/frontend/src/app/components/config/config_update.rs index afb2a5ab0..fb4a87727 100644 --- a/frontend/src/app/components/config/config_update.rs +++ b/frontend/src/app/components/config/config_update.rs @@ -238,20 +238,18 @@ mod tests { #[test] fn update_config_metadata_update_applies_and_cleans_payload() { let mut config = ConfigDto::default(); - let mut metadata_cfg = MetadataUpdateConfigDto { - ffprobe_enabled: true, - ffprobe_timeout: Some(60), - ..MetadataUpdateConfigDto::default() - }; + let mut metadata_cfg = MetadataUpdateConfigDto::default(); + metadata_cfg.ffprobe.enabled = true; + metadata_cfg.ffprobe.timeout = Some(60); assert!(!metadata_cfg.is_empty()); metadata_cfg.clean(); - assert_eq!(metadata_cfg.ffprobe_timeout, None); + assert_eq!(metadata_cfg.ffprobe.timeout, None); update_config(&mut config, vec![ConfigForm::MetadataUpdate(true, metadata_cfg)]); let stored = config.metadata_update.as_ref().expect("metadata_update config should be set"); - assert!(stored.ffprobe_enabled); - assert_eq!(stored.ffprobe_timeout, None); + assert!(stored.ffprobe.enabled); + assert_eq!(stored.ffprobe.timeout, None); } } diff --git a/frontend/src/app/components/config/library_config_view.rs b/frontend/src/app/components/config/library_config_view.rs index cd626529a..088b1dcab 100644 --- a/frontend/src/app/components/config/library_config_view.rs +++ b/frontend/src/app/components/config/library_config_view.rs @@ -11,13 +11,13 @@ use crate::{ }, context::ConfigContext, }, - config_field, config_field_bool, config_field_child, config_field_optional, edit_field_bool, edit_field_list, - edit_field_number, edit_field_number_u64, edit_field_text, edit_field_text_option, generate_form_reducer, + config_field, config_field_bool, config_field_child, edit_field_bool, edit_field_list, edit_field_text, + generate_form_reducer, i18n::use_translation, }; use shared::model::{ LibraryConfigDto, LibraryContentType, LibraryMetadataConfigDto, LibraryMetadataFormat, - LibraryMetadataReadConfigDto, LibraryPlaylistConfigDto, LibraryScanDirectoryDto, LibraryTmdbConfigDto, + LibraryMetadataReadConfigDto, LibraryPlaylistConfigDto, LibraryScanDirectoryDto, }; use std::rc::Rc; use yew::prelude::*; @@ -31,8 +31,6 @@ const LABEL_METADATA_PATH: &str = "LABEL.METADATA_PATH"; const LABEL_PLAYLIST: &str = "LABEL.PLAYLIST"; const LABEL_MOVIE_CATEGORY: &str = "LABEL.MOVIE_CATEGORY"; const LABEL_SERIES_CATEGORY: &str = "LABEL.SERIES_CATEGORY"; -const LABEL_TMDB: &str = "LABEL.TMDB"; -const LABEL_API_KEY: &str = "LABEL.API_KEY"; const LABEL_READ_EXISTING: &str = "LABEL.READ_EXISTING"; const LABEL_KODI: &str = "LABEL.KODI"; const LABEL_JELLYFIN: &str = "LABEL.JELLYFIN"; @@ -45,9 +43,6 @@ const LABEL_CONTENT_TYPE: &str = "LABEL.CONTENT_TYPE"; const LABEL_AUTO: &str = "LABEL.AUTO"; const LABEL_MOVIE: &str = "LABEL.MOVIE"; const LABEL_SERIES: &str = "LABEL.SERIES"; -const LABEL_RATE_LIMIT_MS: &str = "LABEL.RATE_LIMIT_MS"; -const LABEL_CACHE_DURATION_DAYS: &str = "LABEL.CACHE_DURATION_DAYS"; -const LABEL_LANGUAGE: &str = "LABEL.LANGUAGE"; const LABEL_FORMATS: &str = "LABEL.FORMATS"; const LABEL_ADD_FORMAT: &str = "LABEL.ADD_FORMAT"; @@ -94,18 +89,6 @@ generate_form_reducer!( } ); -generate_form_reducer!( - state: LibraryTmdbConfigFormState { form: LibraryTmdbConfigDto }, - action_name: LibraryTmdbConfigFormAction, - fields { - Enabled => enabled: bool, - ApiKey => api_key: Option, - RateLimitMs => rate_limit_ms: u64, - CacheDurationDays => cache_duration_days: u32, - Language => language: String, - } -); - #[component] pub fn LibraryConfigView() -> Html { let translate = use_translation(); @@ -125,15 +108,11 @@ pub fn LibraryConfigView() -> Html { LibraryMetadataReadConfigFormState { form: LibraryMetadataReadConfigDto::default(), modified: false } }); - let tmdb_state: UseReducerHandle = - use_reducer(|| LibraryTmdbConfigFormState { form: LibraryTmdbConfigDto::default(), modified: false }); - { let form_state = form_state.clone(); let playlist_state = playlist_state.clone(); let metadata_state = metadata_state.clone(); let metadata_read_state = metadata_read_state.clone(); - let tmdb_state = tmdb_state.clone(); let library_cfg = config_ctx.config.as_ref().and_then(|c| c.config.library.clone()); use_effect_with((library_cfg, config_view_ctx.edit_mode.clone()), move |(library_cfg, _mode)| { @@ -143,14 +122,12 @@ pub fn LibraryConfigView() -> Html { metadata_state.dispatch(LibraryMetadataConfigFormAction::SetAll(library.metadata.clone())); metadata_read_state .dispatch(LibraryMetadataReadConfigFormAction::SetAll(library.metadata.read_existing.clone())); - tmdb_state.dispatch(LibraryTmdbConfigFormAction::SetAll(library.metadata.tmdb.clone())); } else { form_state.dispatch(LibraryConfigFormAction::SetAll(LibraryConfigDto::default())); playlist_state.dispatch(LibraryPlaylistConfigFormAction::SetAll(LibraryPlaylistConfigDto::default())); metadata_state.dispatch(LibraryMetadataConfigFormAction::SetAll(LibraryMetadataConfigDto::default())); metadata_read_state .dispatch(LibraryMetadataReadConfigFormAction::SetAll(LibraryMetadataReadConfigDto::default())); - tmdb_state.dispatch(LibraryTmdbConfigFormAction::SetAll(LibraryTmdbConfigDto::default())); } || () }); @@ -162,19 +139,15 @@ pub fn LibraryConfigView() -> Html { let playlist_state = playlist_state.clone(); let metadata_state = metadata_state.clone(); let metadata_read_state = metadata_read_state.clone(); - let tmdb_state = tmdb_state.clone(); use_effect_with( - (form_state, playlist_state, metadata_state, metadata_read_state, tmdb_state), - move |(form, playlist, metadata, metadata_read, tmdb)| { + (form_state, playlist_state, metadata_state, metadata_read_state), + move |(form, playlist, metadata, metadata_read)| { let mut new_form = form.form.clone(); new_form.playlist = playlist.form.clone(); new_form.metadata = metadata.form.clone(); new_form.metadata.read_existing = metadata_read.form.clone(); - new_form.metadata.tmdb = tmdb.form.clone(); - - let modified = - form.modified || playlist.modified || metadata.modified || metadata_read.modified || tmdb.modified; + let modified = form.modified || playlist.modified || metadata.modified || metadata_read.modified; on_form_change.emit(ConfigForm::Library(modified, new_form)); }, @@ -379,7 +352,6 @@ pub fn LibraryConfigView() -> Html { let render_view_mode = || { let metadata = &metadata_state.form; let playlist = &playlist_state.form; - let tmdb = &tmdb_state.form; let read_existing = &metadata_read_state.form; html! { @@ -410,15 +382,6 @@ pub fn LibraryConfigView() -> Html { { config_field_bool!(read_existing, translate.t(LABEL_PLEX), plex) } - -

{translate.t(LABEL_TMDB)}

- { config_field_bool!(tmdb, translate.t(LABEL_ENABLED), enabled) } - { config_field_optional!(tmdb, translate.t(LABEL_API_KEY), api_key) } - { config_field!(tmdb, translate.t(LABEL_RATE_LIMIT_MS), rate_limit_ms) } - { config_field!(tmdb, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days) } - { config_field!(tmdb, translate.t(LABEL_LANGUAGE), language) } - -

{translate.t(LABEL_PLAYLIST)}

{ config_field!(playlist, translate.t(LABEL_MOVIE_CATEGORY), movie_category) } @@ -484,15 +447,6 @@ pub fn LibraryConfigView() -> Html { { edit_field_bool!(metadata_read_state, translate.t(LABEL_PLEX), plex, LibraryMetadataReadConfigFormAction::Plex) } - -

{translate.t(LABEL_TMDB)}

- { edit_field_bool!(tmdb_state, translate.t(LABEL_ENABLED), enabled, LibraryTmdbConfigFormAction::Enabled) } - { edit_field_text_option!(tmdb_state, translate.t(LABEL_API_KEY), api_key, LibraryTmdbConfigFormAction::ApiKey, true) } - { edit_field_number_u64!(tmdb_state, translate.t(LABEL_RATE_LIMIT_MS), rate_limit_ms, LibraryTmdbConfigFormAction::RateLimitMs) } - { edit_field_number!(tmdb_state, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days, LibraryTmdbConfigFormAction::CacheDurationDays) } - { edit_field_text!(tmdb_state, translate.t(LABEL_LANGUAGE), language, LibraryTmdbConfigFormAction::Language) } - -

{translate.t(LABEL_PLAYLIST)}

{ edit_field_text!(playlist_state, translate.t(LABEL_MOVIE_CATEGORY), movie_category, LibraryPlaylistConfigFormAction::MovieCategory) } diff --git a/frontend/src/app/components/config/metadata_update_config_view.rs b/frontend/src/app/components/config/metadata_update_config_view.rs index 600663611..2bdd3bd44 100644 --- a/frontend/src/app/components/config/metadata_update_config_view.rs +++ b/frontend/src/app/components/config/metadata_update_config_view.rs @@ -7,14 +7,18 @@ use crate::{ }, dto_field_id, number_input::NumberInput, + Card, }, context::ConfigContext, }, - config_field, config_field_bool, config_field_optional, edit_field_bool, edit_field_number_option_u64, - edit_field_number_u8, edit_field_number_usize, edit_field_text, generate_form_reducer, + config_field, config_field_bool, config_field_optional, edit_field_bool, edit_field_number, + edit_field_number_option_u64, edit_field_number_u64, edit_field_number_u8, edit_field_number_usize, + edit_field_text, edit_field_text_option, generate_form_reducer, i18n::use_translation, }; -use shared::model::MetadataUpdateConfigDto; +use shared::model::{ + FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, TmdbConfigDto, +}; use yew::prelude::*; const LABEL_QUEUE_LOG_INTERVAL: &str = "LABEL.METADATA_QUEUE_LOG_INTERVAL"; @@ -23,6 +27,7 @@ const LABEL_MAX_RESOLVE_RETRY_BACKOFF: &str = "LABEL.METADATA_MAX_RESOLVE_RETRY_ const LABEL_RESOLVE_MIN_RETRY_BASE: &str = "LABEL.METADATA_RESOLVE_MIN_RETRY_BASE"; const LABEL_RESOLVE_EXHAUSTION_RESET_GAP: &str = "LABEL.METADATA_RESOLVE_EXHAUSTION_RESET_GAP"; const LABEL_PROBE_COOLDOWN: &str = "LABEL.METADATA_PROBE_COOLDOWN"; +const LABEL_TMDB_COOLDOWN: &str = "LABEL.METADATA_TMDB_COOLDOWN"; const LABEL_RETRY_DELAY: &str = "LABEL.METADATA_RETRY_DELAY"; const LABEL_PROBE_RETRY_LOAD_RETRY_DELAY: &str = "LABEL.METADATA_PROBE_RETRY_LOAD_RETRY_DELAY"; const LABEL_WORKER_IDLE_TIMEOUT: &str = "LABEL.METADATA_WORKER_IDLE_TIMEOUT"; @@ -39,36 +44,88 @@ const LABEL_FFPROBE_ANALYZE_DURATION: &str = "LABEL.METADATA_FFPROBE_ANALYZE_DUR const LABEL_FFPROBE_PROBE_SIZE: &str = "LABEL.METADATA_FFPROBE_PROBE_SIZE"; const LABEL_FFPROBE_LIVE_ANALYZE_DURATION: &str = "LABEL.METADATA_FFPROBE_LIVE_ANALYZE_DURATION"; const LABEL_FFPROBE_LIVE_PROBE_SIZE: &str = "LABEL.METADATA_FFPROBE_LIVE_PROBE_SIZE"; +const LABEL_TMDB: &str = "LABEL.TMDB"; +const LABEL_LOG: &str = "LABEL.LOG"; +const LABEL_RESOLVE: &str = "LABEL.RESOLVE"; +const LABEL_PROBE: &str = "LABEL.PROBE"; +const LABEL_FFPROBE: &str = "LABEL.FFPROBE"; +const LABEL_SETTINGS: &str = "LABEL.SETTINGS"; +const LABEL_ENABLED: &str = "LABEL.ENABLED"; +const LABEL_API_KEY: &str = "LABEL.API_KEY"; +const LABEL_RATE_LIMIT_MS: &str = "LABEL.RATE_LIMIT_MS"; +const LABEL_CACHE_DURATION_DAYS: &str = "LABEL.CACHE_DURATION_DAYS"; +const LABEL_LANGUAGE: &str = "LABEL.LANGUAGE"; generate_form_reducer!( state: MetadataUpdateConfigFormState { form: MetadataUpdateConfigDto }, action_name: MetadataUpdateConfigFormAction, fields { - FfprobeEnabled => ffprobe_enabled: bool, - FfprobeTimeout => ffprobe_timeout: Option, - FfprobeAnalyzeDuration => ffprobe_analyze_duration: String, - FfprobeProbeSize => ffprobe_probe_size: String, - FfprobeLiveAnalyzeDuration => ffprobe_live_analyze_duration: String, - FfprobeLiveProbeSize => ffprobe_live_probe_size: String, - MaxAttemptsResolve => max_attempts_resolve: u8, - MaxAttemptsProbe => max_attempts_probe: u8, - BackoffJitterPercent => backoff_jitter_percent: u8, - ResolveMinRetryBase => resolve_min_retry_base: String, - MaxResolveRetryBackoff => max_resolve_retry_backoff: String, - ProbeRetryBackoffStep1 => probe_retry_backoff_step_1: String, - ProbeRetryBackoffStep2 => probe_retry_backoff_step_2: String, - ProbeRetryBackoffStep3 => probe_retry_backoff_step_3: String, RetryDelay => retry_delay: String, - ProbeRetryLoadRetryDelay => probe_retry_load_retry_delay: String, - ResolveExhaustionResetGap => resolve_exhaustion_reset_gap: String, - ProbeCooldown => probe_cooldown: String, MaxQueueSize => max_queue_size: usize, - QueueLogInterval => queue_log_interval: String, - ProgressLogInterval => progress_log_interval: String, WorkerIdleTimeout => worker_idle_timeout: String, } ); +generate_form_reducer!( + state: MetadataLogConfigFormState { form: MetadataLogConfigDto }, + action_name: MetadataLogConfigFormAction, + fields { + QueueInterval => queue_interval: String, + ProgressInterval => progress_interval: String, + } +); + +generate_form_reducer!( + state: ResolveConfigFormState { form: ResolveConfigDto }, + action_name: ResolveConfigFormAction, + fields { + MaxRetryBackoff => max_retry_backoff: String, + MinRetryBase => min_retry_base: String, + ExhaustionResetGap => exhaustion_reset_gap: String, + MaxAttempts => max_attempts: u8, + } +); + +generate_form_reducer!( + state: ProbeConfigFormState { form: ProbeConfigDto }, + action_name: ProbeConfigFormAction, + fields { + Cooldown => cooldown: String, + RetryLoadRetryDelay => retry_load_retry_delay: String, + RetryBackoffStep1 => retry_backoff_step_1: String, + RetryBackoffStep2 => retry_backoff_step_2: String, + RetryBackoffStep3 => retry_backoff_step_3: String, + MaxAttempts => max_attempts: u8, + BackoffJitterPercent => backoff_jitter_percent: u8, + } +); + +generate_form_reducer!( + state: FfprobeConfigFormState { form: FfprobeConfigDto }, + action_name: FfprobeConfigFormAction, + fields { + Enabled => enabled: bool, + Timeout => timeout: Option, + AnalyzeDuration => analyze_duration: String, + ProbeSize => probe_size: String, + LiveAnalyzeDuration => live_analyze_duration: String, + LiveProbeSize => live_probe_size: String, + } +); + +generate_form_reducer!( + state: TmdbConfigFormState { form: TmdbConfigDto }, + action_name: TmdbConfigFormAction, + fields { + Enabled => enabled: bool, + ApiKey => api_key: Option, + RateLimitMs => rate_limit_ms: u64, + CacheDurationDays => cache_duration_days: u32, + Language => language: String, + Cooldown => cooldown: String, + } +); + #[component] pub fn MetadataUpdateConfigView() -> Html { let translate = use_translation(); @@ -78,32 +135,74 @@ pub fn MetadataUpdateConfigView() -> Html { let form_state: UseReducerHandle = use_reducer(|| MetadataUpdateConfigFormState { form: MetadataUpdateConfigDto::default(), modified: false }); + let log_state: UseReducerHandle = + use_reducer(|| MetadataLogConfigFormState { form: MetadataLogConfigDto::default(), modified: false }); + let resolve_state: UseReducerHandle = + use_reducer(|| ResolveConfigFormState { form: ResolveConfigDto::default(), modified: false }); + let probe_state: UseReducerHandle = + use_reducer(|| ProbeConfigFormState { form: ProbeConfigDto::default(), modified: false }); + let ffprobe_state: UseReducerHandle = + use_reducer(|| FfprobeConfigFormState { form: FfprobeConfigDto::default(), modified: false }); + let tmdb_state: UseReducerHandle = + use_reducer(|| TmdbConfigFormState { form: TmdbConfigDto::default(), modified: false }); { let on_form_change = config_view_ctx.on_form_change.clone(); - let deps = (form_state.clone(), form_state.modified); - use_effect_with(deps, move |(state, modified)| { - on_form_change.emit(ConfigForm::MetadataUpdate(*modified, state.form.clone())); + let deps = ( + form_state.clone(), + log_state.clone(), + resolve_state.clone(), + probe_state.clone(), + ffprobe_state.clone(), + tmdb_state.clone(), + ); + use_effect_with(deps, move |(form, log, resolve, probe, ffprobe, tmdb)| { + let mut merged = form.form.clone(); + merged.log = log.form.clone(); + merged.resolve = resolve.form.clone(); + merged.probe = probe.form.clone(); + merged.ffprobe = ffprobe.form.clone(); + merged.tmdb = tmdb.form.clone(); + on_form_change.emit(ConfigForm::MetadataUpdate( + form.modified + || log.modified + || resolve.modified + || probe.modified + || ffprobe.modified + || tmdb.modified, + merged, + )); }); } { let form_state = form_state.clone(); + let log_state = log_state.clone(); + let resolve_state = resolve_state.clone(); + let probe_state = probe_state.clone(); + let ffprobe_state = ffprobe_state.clone(); + let tmdb_state = tmdb_state.clone(); + let metadata_update_cfg = config_ctx.config.as_ref().and_then(|c| c.config.metadata_update.clone()).unwrap_or_default(); use_effect_with((metadata_update_cfg, config_view_ctx.edit_mode.clone()), move |(cfg, _mode)| { form_state.dispatch(MetadataUpdateConfigFormAction::SetAll(cfg.clone())); + log_state.dispatch(MetadataLogConfigFormAction::SetAll(cfg.log.clone())); + resolve_state.dispatch(ResolveConfigFormAction::SetAll(cfg.resolve.clone())); + probe_state.dispatch(ProbeConfigFormAction::SetAll(cfg.probe.clone())); + ffprobe_state.dispatch(FfprobeConfigFormAction::SetAll(cfg.ffprobe.clone())); + tmdb_state.dispatch(TmdbConfigFormAction::SetAll(cfg.tmdb.clone())); || () }); } { - let form_state = form_state.clone(); + let probe_state = probe_state.clone(); let backoff_jitter_error = backoff_jitter_error.clone(); - use_effect_with(form_state.form.backoff_jitter_percent, move |value| { - if *value > 100 { - backoff_jitter_error.set(Some("Backoff jitter percent must be between 0 and 100.".to_string())); - form_state.dispatch(MetadataUpdateConfigFormAction::BackoffJitterPercent(100)); + use_effect_with(probe_state.form.backoff_jitter_percent, move |value| { + if *value > 95 { + backoff_jitter_error.set(Some("Backoff jitter percent must be between 0 and 95.".to_string())); + probe_state.dispatch(ProbeConfigFormAction::BackoffJitterPercent(95)); } else { backoff_jitter_error.set(None); } @@ -112,30 +211,67 @@ pub fn MetadataUpdateConfigView() -> Html { } let render_view_mode = || { + let log = &log_state.form; + let resolve = &resolve_state.form; + let probe = &probe_state.form; + let ffprobe = &ffprobe_state.form; + let tmdb = &tmdb_state.form; + html! { <> - { config_field_bool!(form_state.form, translate.t(LABEL_FFPROBE_ENABLED), ffprobe_enabled) } - { config_field_optional!(form_state.form, translate.t(LABEL_FFPROBE_TIMEOUT), ffprobe_timeout) } - { config_field!(form_state.form, translate.t(LABEL_FFPROBE_ANALYZE_DURATION), ffprobe_analyze_duration) } - { config_field!(form_state.form, translate.t(LABEL_FFPROBE_PROBE_SIZE), ffprobe_probe_size) } - { config_field!(form_state.form, translate.t(LABEL_FFPROBE_LIVE_ANALYZE_DURATION), ffprobe_live_analyze_duration) } - { config_field!(form_state.form, translate.t(LABEL_FFPROBE_LIVE_PROBE_SIZE), ffprobe_live_probe_size) } - { config_field!(form_state.form, translate.t(LABEL_MAX_ATTEMPTS_RESOLVE), max_attempts_resolve) } - { config_field!(form_state.form, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts_probe) } - { config_field!(form_state.form, translate.t(LABEL_BACKOFF_JITTER_PERCENT), backoff_jitter_percent) } - { config_field!(form_state.form, translate.t(LABEL_RESOLVE_MIN_RETRY_BASE), resolve_min_retry_base) } - { config_field!(form_state.form, translate.t(LABEL_MAX_RESOLVE_RETRY_BACKOFF), max_resolve_retry_backoff) } - { config_field!(form_state.form, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_1), probe_retry_backoff_step_1) } - { config_field!(form_state.form, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_2), probe_retry_backoff_step_2) } - { config_field!(form_state.form, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_3), probe_retry_backoff_step_3) } - { config_field!(form_state.form, translate.t(LABEL_RETRY_DELAY), retry_delay) } - { config_field!(form_state.form, translate.t(LABEL_PROBE_RETRY_LOAD_RETRY_DELAY), probe_retry_load_retry_delay) } - { config_field!(form_state.form, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), resolve_exhaustion_reset_gap) } - { config_field!(form_state.form, translate.t(LABEL_PROBE_COOLDOWN), probe_cooldown) } - { config_field!(form_state.form, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size) } - { config_field!(form_state.form, translate.t(LABEL_QUEUE_LOG_INTERVAL), queue_log_interval) } - { config_field!(form_state.form, translate.t(LABEL_PROGRESS_LOG_INTERVAL), progress_log_interval) } - { config_field!(form_state.form, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout) } + + +

{translate.t(LABEL_FFPROBE)}

+ { config_field_bool!(ffprobe, translate.t(LABEL_FFPROBE_ENABLED), enabled) } + { config_field_optional!(ffprobe, translate.t(LABEL_FFPROBE_TIMEOUT), timeout) } + { config_field!(ffprobe, translate.t(LABEL_FFPROBE_ANALYZE_DURATION), analyze_duration) } + { config_field!(ffprobe, translate.t(LABEL_FFPROBE_PROBE_SIZE), probe_size) } + { config_field!(ffprobe, translate.t(LABEL_FFPROBE_LIVE_ANALYZE_DURATION), live_analyze_duration) } + { config_field!(ffprobe, translate.t(LABEL_FFPROBE_LIVE_PROBE_SIZE), live_probe_size) } + + + +

{translate.t(LABEL_TMDB)}

+ { config_field_bool!(tmdb, translate.t(LABEL_ENABLED), enabled) } + { config_field_optional!(tmdb, translate.t(LABEL_API_KEY), api_key) } + { config_field!(tmdb, translate.t(LABEL_RATE_LIMIT_MS), rate_limit_ms) } + { config_field!(tmdb, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days) } + { config_field!(tmdb, translate.t(LABEL_LANGUAGE), language) } + { config_field!(tmdb, translate.t(LABEL_TMDB_COOLDOWN), cooldown) } + + + +

{translate.t(LABEL_SETTINGS)}

+ { config_field!(form_state.form, translate.t(LABEL_RETRY_DELAY), retry_delay) } + { config_field!(form_state.form, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size) } + { config_field!(form_state.form, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout) } + + + +

{translate.t(LABEL_LOG)}

+ { config_field!(log, translate.t(LABEL_QUEUE_LOG_INTERVAL), queue_interval) } + { config_field!(log, translate.t(LABEL_PROGRESS_LOG_INTERVAL), progress_interval) } + + + +

{translate.t(LABEL_RESOLVE)}

+ { config_field!(resolve, translate.t(LABEL_MAX_ATTEMPTS_RESOLVE), max_attempts) } + { config_field!(resolve, translate.t(LABEL_RESOLVE_MIN_RETRY_BASE), min_retry_base) } + { config_field!(resolve, translate.t(LABEL_MAX_RESOLVE_RETRY_BACKOFF), max_retry_backoff) } + { config_field!(resolve, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), exhaustion_reset_gap) } + + + +

{translate.t(LABEL_PROBE)}

+ { config_field!(probe, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts) } + { config_field!(probe, translate.t(LABEL_BACKOFF_JITTER_PERCENT), backoff_jitter_percent) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_1), retry_backoff_step_1) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_2), retry_backoff_step_2) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_3), retry_backoff_step_3) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_LOAD_RETRY_DELAY), retry_load_retry_delay) } + { config_field!(probe, translate.t(LABEL_PROBE_COOLDOWN), cooldown) } + + } }; @@ -143,65 +279,96 @@ pub fn MetadataUpdateConfigView() -> Html { let render_edit_mode = || { let jitter_error_text = (*backoff_jitter_error).clone(); let jitter_error_state = backoff_jitter_error.clone(); - let jitter_form_state = form_state.clone(); + let jitter_probe_state = probe_state.clone(); let jitter_label = translate.t(LABEL_BACKOFF_JITTER_PERCENT); - let jitter_field_id = dto_field_id(&jitter_form_state.form, "backoff_jitter_percent"); + let jitter_field_id = dto_field_id(&jitter_probe_state.form, "backoff_jitter_percent"); html! { <> - { edit_field_bool!(form_state, translate.t(LABEL_FFPROBE_ENABLED), ffprobe_enabled, MetadataUpdateConfigFormAction::FfprobeEnabled) } - { edit_field_number_option_u64!(form_state, translate.t(LABEL_FFPROBE_TIMEOUT), ffprobe_timeout, MetadataUpdateConfigFormAction::FfprobeTimeout) } - { edit_field_text!(form_state, translate.t(LABEL_FFPROBE_ANALYZE_DURATION), ffprobe_analyze_duration, MetadataUpdateConfigFormAction::FfprobeAnalyzeDuration) } - { edit_field_text!(form_state, translate.t(LABEL_FFPROBE_PROBE_SIZE), ffprobe_probe_size, MetadataUpdateConfigFormAction::FfprobeProbeSize) } - { edit_field_text!(form_state, translate.t(LABEL_FFPROBE_LIVE_ANALYZE_DURATION), ffprobe_live_analyze_duration, MetadataUpdateConfigFormAction::FfprobeLiveAnalyzeDuration) } - { edit_field_text!(form_state, translate.t(LABEL_FFPROBE_LIVE_PROBE_SIZE), ffprobe_live_probe_size, MetadataUpdateConfigFormAction::FfprobeLiveProbeSize) } - { edit_field_number_u8!(form_state, translate.t(LABEL_MAX_ATTEMPTS_RESOLVE), max_attempts_resolve, MetadataUpdateConfigFormAction::MaxAttemptsResolve) } - { edit_field_number_u8!(form_state, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts_probe, MetadataUpdateConfigFormAction::MaxAttemptsProbe) } -
- | { - match value { - Some(raw) if !(0..=100).contains(&raw) => { - jitter_error_state.set(Some("Backoff jitter percent must be between 0 and 100.".to_string())); - } - Some(raw) => { - jitter_error_state.set(None); - if let Ok(parsed) = u8::try_from(raw) { - jitter_form_state.dispatch(MetadataUpdateConfigFormAction::BackoffJitterPercent(parsed)); + + +

{translate.t(LABEL_FFPROBE)}

+ { edit_field_bool!(ffprobe_state, translate.t(LABEL_FFPROBE_ENABLED), enabled, FfprobeConfigFormAction::Enabled) } + { edit_field_number_option_u64!(ffprobe_state, translate.t(LABEL_FFPROBE_TIMEOUT), timeout, FfprobeConfigFormAction::Timeout) } + { edit_field_text!(ffprobe_state, translate.t(LABEL_FFPROBE_ANALYZE_DURATION), analyze_duration, FfprobeConfigFormAction::AnalyzeDuration) } + { edit_field_text!(ffprobe_state, translate.t(LABEL_FFPROBE_PROBE_SIZE), probe_size, FfprobeConfigFormAction::ProbeSize) } + { edit_field_text!(ffprobe_state, translate.t(LABEL_FFPROBE_LIVE_ANALYZE_DURATION), live_analyze_duration, FfprobeConfigFormAction::LiveAnalyzeDuration) } + { edit_field_text!(ffprobe_state, translate.t(LABEL_FFPROBE_LIVE_PROBE_SIZE), live_probe_size, FfprobeConfigFormAction::LiveProbeSize) } + + + +

{translate.t(LABEL_TMDB)}

+ { edit_field_bool!(tmdb_state, translate.t(LABEL_ENABLED), enabled, TmdbConfigFormAction::Enabled) } + { edit_field_text_option!(tmdb_state, translate.t(LABEL_API_KEY), api_key, TmdbConfigFormAction::ApiKey, true) } + { edit_field_number_u64!(tmdb_state, translate.t(LABEL_RATE_LIMIT_MS), rate_limit_ms, TmdbConfigFormAction::RateLimitMs) } + { edit_field_number!(tmdb_state, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days, TmdbConfigFormAction::CacheDurationDays) } + { edit_field_text!(tmdb_state, translate.t(LABEL_LANGUAGE), language, TmdbConfigFormAction::Language) } + { edit_field_text!(tmdb_state, translate.t(LABEL_TMDB_COOLDOWN), cooldown, TmdbConfigFormAction::Cooldown) } + + + +

{translate.t(LABEL_SETTINGS)}

+ { edit_field_text!(form_state, translate.t(LABEL_RETRY_DELAY), retry_delay, MetadataUpdateConfigFormAction::RetryDelay) } + { edit_field_number_usize!(form_state, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size, MetadataUpdateConfigFormAction::MaxQueueSize) } + { edit_field_text!(form_state, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout, MetadataUpdateConfigFormAction::WorkerIdleTimeout) } + + + +

{translate.t(LABEL_LOG)}

+ { edit_field_text!(log_state, translate.t(LABEL_QUEUE_LOG_INTERVAL), queue_interval, MetadataLogConfigFormAction::QueueInterval) } + { edit_field_text!(log_state, translate.t(LABEL_PROGRESS_LOG_INTERVAL), progress_interval, MetadataLogConfigFormAction::ProgressInterval) } + + + +

{translate.t(LABEL_RESOLVE)}

+ { edit_field_number_u8!(resolve_state, translate.t(LABEL_MAX_ATTEMPTS_RESOLVE), max_attempts, ResolveConfigFormAction::MaxAttempts) } + { edit_field_text!(resolve_state, translate.t(LABEL_RESOLVE_MIN_RETRY_BASE), min_retry_base, ResolveConfigFormAction::MinRetryBase) } + { edit_field_text!(resolve_state, translate.t(LABEL_MAX_RESOLVE_RETRY_BACKOFF), max_retry_backoff, ResolveConfigFormAction::MaxRetryBackoff) } + { edit_field_text!(resolve_state, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), exhaustion_reset_gap, ResolveConfigFormAction::ExhaustionResetGap) } + + + +

{translate.t(LABEL_PROBE)}

+ { edit_field_number_u8!(probe_state, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts, ProbeConfigFormAction::MaxAttempts) } +
+ | { + match value { + Some(raw) if !(0..=95).contains(&raw) => { + jitter_error_state.set(Some("Backoff jitter percent must be between 0 and 95.".to_string())); + } + Some(raw) => { + jitter_error_state.set(None); + if let Ok(parsed) = u8::try_from(raw) { + jitter_probe_state.dispatch(ProbeConfigFormAction::BackoffJitterPercent(parsed)); + } + } + None => { + jitter_error_state.set(None); + jitter_probe_state.dispatch(ProbeConfigFormAction::BackoffJitterPercent(0)); } } - None => { - jitter_error_state.set(None); - jitter_form_state.dispatch(MetadataUpdateConfigFormAction::BackoffJitterPercent(0)); - } + })} + /> + { + if let Some(error_text) = jitter_error_text { + html! {
{error_text}
} + } else { + html! {} } - })} - /> - { - if let Some(error_text) = jitter_error_text { - html! {
{error_text}
} - } else { - html! {} } - } -
- { edit_field_text!(form_state, translate.t(LABEL_RESOLVE_MIN_RETRY_BASE), resolve_min_retry_base, MetadataUpdateConfigFormAction::ResolveMinRetryBase) } - { edit_field_text!(form_state, translate.t(LABEL_MAX_RESOLVE_RETRY_BACKOFF), max_resolve_retry_backoff, MetadataUpdateConfigFormAction::MaxResolveRetryBackoff) } - { edit_field_text!(form_state, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_1), probe_retry_backoff_step_1, MetadataUpdateConfigFormAction::ProbeRetryBackoffStep1) } - { edit_field_text!(form_state, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_2), probe_retry_backoff_step_2, MetadataUpdateConfigFormAction::ProbeRetryBackoffStep2) } - { edit_field_text!(form_state, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_3), probe_retry_backoff_step_3, MetadataUpdateConfigFormAction::ProbeRetryBackoffStep3) } - { edit_field_text!(form_state, translate.t(LABEL_RETRY_DELAY), retry_delay, MetadataUpdateConfigFormAction::RetryDelay) } - { edit_field_text!(form_state, translate.t(LABEL_PROBE_RETRY_LOAD_RETRY_DELAY), probe_retry_load_retry_delay, MetadataUpdateConfigFormAction::ProbeRetryLoadRetryDelay) } - { edit_field_text!(form_state, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), resolve_exhaustion_reset_gap, MetadataUpdateConfigFormAction::ResolveExhaustionResetGap) } - { edit_field_text!(form_state, translate.t(LABEL_PROBE_COOLDOWN), probe_cooldown, MetadataUpdateConfigFormAction::ProbeCooldown) } - { edit_field_number_usize!(form_state, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size, MetadataUpdateConfigFormAction::MaxQueueSize) } - { edit_field_text!(form_state, translate.t(LABEL_QUEUE_LOG_INTERVAL), queue_log_interval, MetadataUpdateConfigFormAction::QueueLogInterval) } - { edit_field_text!(form_state, translate.t(LABEL_PROGRESS_LOG_INTERVAL), progress_log_interval, MetadataUpdateConfigFormAction::ProgressLogInterval) } - { edit_field_text!(form_state, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout, MetadataUpdateConfigFormAction::WorkerIdleTimeout) } +
+ { edit_field_text!(probe_state, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_1), retry_backoff_step_1, ProbeConfigFormAction::RetryBackoffStep1) } + { edit_field_text!(probe_state, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_2), retry_backoff_step_2, ProbeConfigFormAction::RetryBackoffStep2) } + { edit_field_text!(probe_state, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_3), retry_backoff_step_3, ProbeConfigFormAction::RetryBackoffStep3) } + { edit_field_text!(probe_state, translate.t(LABEL_PROBE_RETRY_LOAD_RETRY_DELAY), retry_load_retry_delay, ProbeConfigFormAction::RetryLoadRetryDelay) } + { edit_field_text!(probe_state, translate.t(LABEL_PROBE_COOLDOWN), cooldown, ProbeConfigFormAction::Cooldown) } + + } }; diff --git a/frontend/src/app/components/field_explanation.rs b/frontend/src/app/components/field_explanation.rs index 339d85745..00ef433b5 100644 --- a/frontend/src/app/components/field_explanation.rs +++ b/frontend/src/app/components/field_explanation.rs @@ -109,10 +109,14 @@ pub fn show_field_explanation(field_id: &str, field_label: &str, dialog: &Dialog .content( html! {
-

{title}

- {for explanation_paragraphs.into_iter().map(|paragraph| { - html! {

{paragraph}

} - })} +
+

{title}

+
+
+ {for explanation_paragraphs.into_iter().map(|paragraph| { + html! {

{paragraph}

} + })} +
}, Some(actions), diff --git a/frontend/src/app/components/setup/setup_helpers.rs b/frontend/src/app/components/setup/setup_helpers.rs index ff7a9fb9a..8e5ce5fca 100644 --- a/frontend/src/app/components/setup/setup_helpers.rs +++ b/frontend/src/app/components/setup/setup_helpers.rs @@ -658,8 +658,9 @@ mod tests { #[test] fn setup_metadata_update_empty_form_clears_existing_payload() { let mut app_config = AppConfigDto::default(); - app_config.config.metadata_update = - Some(MetadataUpdateConfigDto { ffprobe_enabled: true, ..Default::default() }); + let mut metadata_update = MetadataUpdateConfigDto::default(); + metadata_update.ffprobe.enabled = true; + app_config.config.metadata_update = Some(metadata_update); let config_ctx = ConfigContext { config: Some(Rc::new(app_config)), api_proxy: None }; let mut form_state = crate::app::components::setup::SetupConfigFormState::default(); form_state.update_form(ConfigForm::MetadataUpdate(true, MetadataUpdateConfigDto::default())); @@ -672,14 +673,14 @@ mod tests { fn setup_metadata_update_form_applies_cleaned_payload() { let config_ctx = ConfigContext { config: Some(Rc::new(AppConfigDto::default())), api_proxy: None }; let mut form_state = crate::app::components::setup::SetupConfigFormState::default(); - form_state.update_form(ConfigForm::MetadataUpdate( - true, - MetadataUpdateConfigDto { ffprobe_enabled: true, ffprobe_timeout: Some(60), ..Default::default() }, - )); + let mut metadata_update = MetadataUpdateConfigDto::default(); + metadata_update.ffprobe.enabled = true; + metadata_update.ffprobe.timeout = Some(60); + form_state.update_form(ConfigForm::MetadataUpdate(true, metadata_update)); let app_cfg = build_setup_app_config(&config_ctx, &form_state, SourcesConfigDto::default()); let metadata_update = app_cfg.config.metadata_update.expect("metadata_update config should be present"); - assert!(metadata_update.ffprobe_enabled); - assert_eq!(metadata_update.ffprobe_timeout, None); + assert!(metadata_update.ffprobe.enabled); + assert_eq!(metadata_update.ffprobe.timeout, None); } } diff --git a/shared/src/model/config/library.rs b/shared/src/model/config/library.rs index 9174b45d8..45fca44f9 100644 --- a/shared/src/model/config/library.rs +++ b/shared/src/model/config/library.rs @@ -3,10 +3,8 @@ use crate::{ info_err_res, utils::{ default_as_true, default_metadata_path, default_movie_category, default_series_category, - default_storage_formats, default_supported_library_extensions, default_tmdb_api_key, - default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_rate_limit_ms, - is_default_supported_library_extensions, is_default_tmdb_cache_duration_days, is_default_tmdb_language, - is_default_tmdb_rate_limit_ms, is_tmdb_default_api_key, is_true, TMDB_API_KEY, + default_storage_formats, default_supported_library_extensions, is_default_supported_library_extensions, + is_true, }, }; use serde::{Deserialize, Serialize}; @@ -77,8 +75,6 @@ pub struct LibraryMetadataConfigDto { pub path: String, #[serde(default)] pub read_existing: LibraryMetadataReadConfigDto, - #[serde(default)] - pub tmdb: LibraryTmdbConfigDto, #[serde(default = "default_as_true")] pub fallback_to_filename: bool, #[serde(default = "default_storage_formats", skip_serializing_if = "Vec::is_empty")] @@ -90,14 +86,12 @@ impl LibraryMetadataConfigDto { self.fallback_to_filename && self.path == default_metadata_path() && self.read_existing.is_empty() - && self.tmdb.is_empty() && self.formats.is_empty() } pub fn clean(&mut self) { if self.path.trim().is_empty() { self.path = default_metadata_path(); } - self.tmdb.clean(); } } @@ -116,36 +110,6 @@ impl LibraryMetadataReadConfigDto { pub fn is_empty(&self) -> bool { self.kodi && self.jellyfin && self.plex } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct LibraryTmdbConfigDto { - #[serde(default)] - pub enabled: bool, - #[serde(default = "default_tmdb_api_key", skip_serializing_if = "is_tmdb_default_api_key")] - pub api_key: Option, - #[serde(default = "default_tmdb_rate_limit_ms", skip_serializing_if = "is_default_tmdb_rate_limit_ms")] - pub rate_limit_ms: u64, - #[serde(default = "default_tmdb_cache_duration_days", skip_serializing_if = "is_default_tmdb_cache_duration_days")] - pub cache_duration_days: u32, - #[serde(default = "default_tmdb_language", skip_serializing_if = "is_default_tmdb_language")] - pub language: String, -} - -impl LibraryTmdbConfigDto { - pub fn is_empty(&self) -> bool { - !self.enabled - && self.api_key.as_ref().is_none_or(|api_key| api_key == TMDB_API_KEY) - && self.rate_limit_ms == default_tmdb_rate_limit_ms() - && self.cache_duration_days == default_tmdb_cache_duration_days() - && self.language == default_tmdb_language() - } - pub fn clean(&mut self) { - if self.api_key.as_ref().is_some_and(|api_key| api_key == TMDB_API_KEY) { - self.api_key = None; - } - } -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum LibraryMetadataFormat { diff --git a/shared/src/model/config/metadata_update.rs b/shared/src/model/config/metadata_update.rs index 0ab1205b3..860dbe73f 100644 --- a/shared/src/model/config/metadata_update.rs +++ b/shared/src/model/config/metadata_update.rs @@ -11,21 +11,23 @@ use crate::{ default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, - default_metadata_retry_delay, default_metadata_worker_idle_timeout, deserialize_as_string, - is_default_metadata_backoff_jitter_percent, is_default_metadata_ffprobe_analyze_duration, - is_default_metadata_ffprobe_live_analyze_duration, is_default_metadata_ffprobe_live_probe_size, - is_default_metadata_ffprobe_probe_size, is_default_metadata_max_attempts_probe, - is_default_metadata_max_attempts_resolve, is_default_metadata_max_queue_size, - is_default_metadata_max_resolve_retry_backoff, is_default_metadata_probe_cooldown, - is_default_metadata_probe_retry_backoff_step_1, is_default_metadata_probe_retry_backoff_step_2, - is_default_metadata_probe_retry_backoff_step_3, is_default_metadata_probe_retry_load_retry_delay, - is_default_metadata_progress_log_interval, is_default_metadata_queue_log_interval, - is_default_metadata_resolve_exhaustion_reset_gap, is_default_metadata_resolve_min_retry_base, - is_default_metadata_retry_delay, is_default_metadata_worker_idle_timeout, is_false, parse_duration_seconds, - parse_size_base_2, + default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, + default_tmdb_api_key, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_rate_limit_ms, + deserialize_as_string, is_default_metadata_backoff_jitter_percent, + is_default_metadata_ffprobe_analyze_duration, is_default_metadata_ffprobe_live_analyze_duration, + is_default_metadata_ffprobe_live_probe_size, is_default_metadata_ffprobe_probe_size, + is_default_metadata_max_attempts_probe, is_default_metadata_max_attempts_resolve, + is_default_metadata_max_queue_size, is_default_metadata_max_resolve_retry_backoff, + is_default_metadata_probe_cooldown, is_default_metadata_probe_retry_backoff_step_1, + is_default_metadata_probe_retry_backoff_step_2, is_default_metadata_probe_retry_backoff_step_3, + is_default_metadata_probe_retry_load_retry_delay, is_default_metadata_progress_log_interval, + is_default_metadata_queue_log_interval, is_default_metadata_resolve_exhaustion_reset_gap, + is_default_metadata_resolve_min_retry_base, is_default_metadata_retry_delay, is_default_metadata_tmdb_cooldown, + is_default_metadata_worker_idle_timeout, is_default_tmdb_cache_duration_days, is_default_tmdb_language, + is_default_tmdb_rate_limit_ms, is_false, is_tmdb_default_api_key, parse_duration_seconds, parse_size_base_2, + TMDB_API_KEY, }, }; -use std::sync::OnceLock; const MIN_DURATION_SECS: u64 = 1; const MIN_ATTEMPTS: u8 = 1; @@ -36,245 +38,449 @@ const DEFAULT_FFPROBE_TIMEOUT_SECS: u64 = 60; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct MetadataUpdateConfigDto { - #[serde( - default = "default_metadata_queue_log_interval", - skip_serializing_if = "is_default_metadata_queue_log_interval" - )] - pub queue_log_interval: String, - #[serde( - default = "default_metadata_progress_log_interval", - skip_serializing_if = "is_default_metadata_progress_log_interval" - )] - pub progress_log_interval: String, - #[serde( - default = "default_metadata_max_resolve_retry_backoff", - skip_serializing_if = "is_default_metadata_max_resolve_retry_backoff" - )] - pub max_resolve_retry_backoff: String, - #[serde( - default = "default_metadata_resolve_min_retry_base", - skip_serializing_if = "is_default_metadata_resolve_min_retry_base" - )] - pub resolve_min_retry_base: String, - #[serde( - default = "default_metadata_resolve_exhaustion_reset_gap", - skip_serializing_if = "is_default_metadata_resolve_exhaustion_reset_gap" - )] - pub resolve_exhaustion_reset_gap: String, - #[serde(default = "default_metadata_probe_cooldown", skip_serializing_if = "is_default_metadata_probe_cooldown")] - pub probe_cooldown: String, + #[serde(default, skip_serializing_if = "MetadataLogConfigDto::is_empty")] + pub log: MetadataLogConfigDto, + #[serde(default, skip_serializing_if = "ResolveConfigDto::is_empty")] + pub resolve: ResolveConfigDto, + #[serde(default, skip_serializing_if = "ProbeConfigDto::is_empty")] + pub probe: ProbeConfigDto, + #[serde(default, skip_serializing_if = "FfprobeConfigDto::is_empty")] + pub ffprobe: FfprobeConfigDto, + #[serde(default, skip_serializing_if = "TmdbConfigDto::is_empty")] + pub tmdb: TmdbConfigDto, #[serde(default = "default_metadata_retry_delay", skip_serializing_if = "is_default_metadata_retry_delay")] pub retry_delay: String, - #[serde( - default = "default_metadata_probe_retry_load_retry_delay", - skip_serializing_if = "is_default_metadata_probe_retry_load_retry_delay" - )] - pub probe_retry_load_retry_delay: String, #[serde( default = "default_metadata_worker_idle_timeout", skip_serializing_if = "is_default_metadata_worker_idle_timeout" )] pub worker_idle_timeout: String, - #[serde( - default = "default_metadata_probe_retry_backoff_step_1", - skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_1" - )] - pub probe_retry_backoff_step_1: String, - #[serde( - default = "default_metadata_probe_retry_backoff_step_2", - skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_2" - )] - pub probe_retry_backoff_step_2: String, - #[serde( - default = "default_metadata_probe_retry_backoff_step_3", - skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_3" - )] - pub probe_retry_backoff_step_3: String, - #[serde( - default = "default_metadata_max_attempts_resolve", - skip_serializing_if = "is_default_metadata_max_attempts_resolve" - )] - pub max_attempts_resolve: u8, - #[serde( - default = "default_metadata_max_attempts_probe", - skip_serializing_if = "is_default_metadata_max_attempts_probe" - )] - pub max_attempts_probe: u8, - #[serde( - default = "default_metadata_backoff_jitter_percent", - skip_serializing_if = "is_default_metadata_backoff_jitter_percent" - )] - pub backoff_jitter_percent: u8, #[serde(default = "default_metadata_max_queue_size", skip_serializing_if = "is_default_metadata_max_queue_size")] pub max_queue_size: usize, - #[serde(default, skip_serializing_if = "is_false")] - pub ffprobe_enabled: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub ffprobe_timeout: Option, - #[serde( - default = "default_metadata_ffprobe_analyze_duration", - skip_serializing_if = "is_default_metadata_ffprobe_analyze_duration", - deserialize_with = "deserialize_as_string" - )] - pub ffprobe_analyze_duration: String, - #[serde( - default = "default_metadata_ffprobe_probe_size", - skip_serializing_if = "is_default_metadata_ffprobe_probe_size", - deserialize_with = "deserialize_as_string" - )] - pub ffprobe_probe_size: String, - #[serde( - default = "default_metadata_ffprobe_live_analyze_duration", - skip_serializing_if = "is_default_metadata_ffprobe_live_analyze_duration", - deserialize_with = "deserialize_as_string" - )] - pub ffprobe_live_analyze_duration: String, - #[serde( - default = "default_metadata_ffprobe_live_probe_size", - skip_serializing_if = "is_default_metadata_ffprobe_live_probe_size", - deserialize_with = "deserialize_as_string" - )] - pub ffprobe_live_probe_size: String, } impl Default for MetadataUpdateConfigDto { fn default() -> Self { Self { - queue_log_interval: default_metadata_queue_log_interval(), - progress_log_interval: default_metadata_progress_log_interval(), - max_resolve_retry_backoff: default_metadata_max_resolve_retry_backoff(), - resolve_min_retry_base: default_metadata_resolve_min_retry_base(), - resolve_exhaustion_reset_gap: default_metadata_resolve_exhaustion_reset_gap(), - probe_cooldown: default_metadata_probe_cooldown(), + log: MetadataLogConfigDto::default(), + resolve: ResolveConfigDto::default(), + probe: ProbeConfigDto::default(), + ffprobe: FfprobeConfigDto::default(), + tmdb: TmdbConfigDto::default(), retry_delay: default_metadata_retry_delay(), - probe_retry_load_retry_delay: default_metadata_probe_retry_load_retry_delay(), worker_idle_timeout: default_metadata_worker_idle_timeout(), - probe_retry_backoff_step_1: default_metadata_probe_retry_backoff_step_1(), - probe_retry_backoff_step_2: default_metadata_probe_retry_backoff_step_2(), - probe_retry_backoff_step_3: default_metadata_probe_retry_backoff_step_3(), - max_attempts_resolve: default_metadata_max_attempts_resolve(), - max_attempts_probe: default_metadata_max_attempts_probe(), - backoff_jitter_percent: default_metadata_backoff_jitter_percent(), max_queue_size: default_metadata_max_queue_size(), - ffprobe_enabled: false, - ffprobe_timeout: None, - ffprobe_analyze_duration: default_metadata_ffprobe_analyze_duration(), - ffprobe_probe_size: default_metadata_ffprobe_probe_size(), - ffprobe_live_analyze_duration: default_metadata_ffprobe_live_analyze_duration(), - ffprobe_live_probe_size: default_metadata_ffprobe_live_probe_size(), } } } -impl MetadataUpdateConfigDto { - fn defaults() -> &'static Self { - static DEFAULTS: OnceLock = OnceLock::new(); - DEFAULTS.get_or_init(Self::default) +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MetadataLogConfigDto { + #[serde( + default = "default_metadata_queue_log_interval", + skip_serializing_if = "is_default_metadata_queue_log_interval" + )] + pub queue_interval: String, + #[serde( + default = "default_metadata_progress_log_interval", + skip_serializing_if = "is_default_metadata_progress_log_interval" + )] + pub progress_interval: String, +} + +impl Default for MetadataLogConfigDto { + fn default() -> Self { + Self { + queue_interval: default_metadata_queue_log_interval(), + progress_interval: default_metadata_progress_log_interval(), + } + } +} + +impl MetadataLogConfigDto { + pub fn is_empty(&self) -> bool { + self.queue_interval == default_metadata_queue_log_interval() + && self.progress_interval == default_metadata_progress_log_interval() } - pub fn is_empty(&self) -> bool { self == Self::defaults() } + fn prepare(&mut self) -> Result<(), TuliproxError> { + let queue_interval_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.queue_interval, + MIN_DURATION_SECS, + "log.queue_interval", + )?; + self.queue_interval = MetadataUpdateConfigDto::canonicalize_seconds(queue_interval_secs); - pub fn clean(&mut self) { - if self.ffprobe_timeout.is_some_and(|v| v == DEFAULT_FFPROBE_TIMEOUT_SECS) { - self.ffprobe_timeout = None; + let progress_interval_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.progress_interval, + MIN_DURATION_SECS, + "log.progress_interval", + )?; + self.progress_interval = MetadataUpdateConfigDto::canonicalize_seconds(progress_interval_secs); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ResolveConfigDto { + #[serde( + default = "default_metadata_max_resolve_retry_backoff", + skip_serializing_if = "is_default_metadata_max_resolve_retry_backoff" + )] + pub max_retry_backoff: String, + #[serde( + default = "default_metadata_resolve_min_retry_base", + skip_serializing_if = "is_default_metadata_resolve_min_retry_base" + )] + pub min_retry_base: String, + #[serde( + default = "default_metadata_resolve_exhaustion_reset_gap", + skip_serializing_if = "is_default_metadata_resolve_exhaustion_reset_gap" + )] + pub exhaustion_reset_gap: String, + #[serde( + default = "default_metadata_max_attempts_resolve", + skip_serializing_if = "is_default_metadata_max_attempts_resolve" + )] + pub max_attempts: u8, +} + +impl Default for ResolveConfigDto { + fn default() -> Self { + Self { + max_retry_backoff: default_metadata_max_resolve_retry_backoff(), + min_retry_base: default_metadata_resolve_min_retry_base(), + exhaustion_reset_gap: default_metadata_resolve_exhaustion_reset_gap(), + max_attempts: default_metadata_max_attempts_resolve(), + } + } +} + +impl ResolveConfigDto { + pub fn is_empty(&self) -> bool { + self.max_retry_backoff == default_metadata_max_resolve_retry_backoff() + && self.min_retry_base == default_metadata_resolve_min_retry_base() + && self.exhaustion_reset_gap == default_metadata_resolve_exhaustion_reset_gap() + && self.max_attempts == default_metadata_max_attempts_resolve() + } + + fn prepare(&mut self) -> Result<(), TuliproxError> { + let max_retry_backoff_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.max_retry_backoff, + MIN_DURATION_SECS, + "resolve.max_retry_backoff", + )?; + self.max_retry_backoff = MetadataUpdateConfigDto::canonicalize_seconds(max_retry_backoff_secs); + + let min_retry_base_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.min_retry_base, + MIN_DURATION_SECS, + "resolve.min_retry_base", + )?; + self.min_retry_base = MetadataUpdateConfigDto::canonicalize_seconds(min_retry_base_secs); + + let exhaustion_reset_gap_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.exhaustion_reset_gap, + MIN_DURATION_SECS, + "resolve.exhaustion_reset_gap", + )?; + self.exhaustion_reset_gap = MetadataUpdateConfigDto::canonicalize_seconds(exhaustion_reset_gap_secs); + + self.max_attempts = self.max_attempts.max(MIN_ATTEMPTS); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ProbeConfigDto { + #[serde(default = "default_metadata_probe_cooldown", skip_serializing_if = "is_default_metadata_probe_cooldown")] + pub cooldown: String, + #[serde( + default = "default_metadata_probe_retry_load_retry_delay", + skip_serializing_if = "is_default_metadata_probe_retry_load_retry_delay" + )] + pub retry_load_retry_delay: String, + #[serde( + default = "default_metadata_probe_retry_backoff_step_1", + skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_1" + )] + pub retry_backoff_step_1: String, + #[serde( + default = "default_metadata_probe_retry_backoff_step_2", + skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_2" + )] + pub retry_backoff_step_2: String, + #[serde( + default = "default_metadata_probe_retry_backoff_step_3", + skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_3" + )] + pub retry_backoff_step_3: String, + #[serde( + default = "default_metadata_max_attempts_probe", + skip_serializing_if = "is_default_metadata_max_attempts_probe" + )] + pub max_attempts: u8, + #[serde( + default = "default_metadata_backoff_jitter_percent", + skip_serializing_if = "is_default_metadata_backoff_jitter_percent" + )] + pub backoff_jitter_percent: u8, +} + +impl Default for ProbeConfigDto { + fn default() -> Self { + Self { + cooldown: default_metadata_probe_cooldown(), + retry_load_retry_delay: default_metadata_probe_retry_load_retry_delay(), + retry_backoff_step_1: default_metadata_probe_retry_backoff_step_1(), + retry_backoff_step_2: default_metadata_probe_retry_backoff_step_2(), + retry_backoff_step_3: default_metadata_probe_retry_backoff_step_3(), + max_attempts: default_metadata_max_attempts_probe(), + backoff_jitter_percent: default_metadata_backoff_jitter_percent(), + } + } +} + +impl ProbeConfigDto { + pub fn is_empty(&self) -> bool { + self.cooldown == default_metadata_probe_cooldown() + && self.retry_load_retry_delay == default_metadata_probe_retry_load_retry_delay() + && self.retry_backoff_step_1 == default_metadata_probe_retry_backoff_step_1() + && self.retry_backoff_step_2 == default_metadata_probe_retry_backoff_step_2() + && self.retry_backoff_step_3 == default_metadata_probe_retry_backoff_step_3() + && self.max_attempts == default_metadata_max_attempts_probe() + && self.backoff_jitter_percent == default_metadata_backoff_jitter_percent() + } + + fn prepare(&mut self) -> Result<(), TuliproxError> { + let cooldown_secs = + MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "probe.cooldown")?; + self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); + + let retry_load_retry_delay_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_load_retry_delay, + MIN_DURATION_SECS, + "probe.retry_load_retry_delay", + )?; + self.retry_load_retry_delay = MetadataUpdateConfigDto::canonicalize_seconds(retry_load_retry_delay_secs); + + let retry_backoff_step_1_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_backoff_step_1, + MIN_DURATION_SECS, + "probe.retry_backoff_step_1", + )?; + self.retry_backoff_step_1 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_1_secs); + + let retry_backoff_step_2_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_backoff_step_2, + MIN_DURATION_SECS, + "probe.retry_backoff_step_2", + )?; + self.retry_backoff_step_2 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_2_secs); + + let retry_backoff_step_3_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_backoff_step_3, + MIN_DURATION_SECS, + "probe.retry_backoff_step_3", + )?; + self.retry_backoff_step_3 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_3_secs); + + self.max_attempts = self.max_attempts.max(MIN_ATTEMPTS); + self.backoff_jitter_percent = self.backoff_jitter_percent.min(MAX_JITTER_PERCENT); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct FfprobeConfigDto { + #[serde(default, skip_serializing_if = "is_false")] + pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde( + default = "default_metadata_ffprobe_analyze_duration", + skip_serializing_if = "is_default_metadata_ffprobe_analyze_duration", + deserialize_with = "deserialize_as_string" + )] + pub analyze_duration: String, + #[serde( + default = "default_metadata_ffprobe_probe_size", + skip_serializing_if = "is_default_metadata_ffprobe_probe_size", + deserialize_with = "deserialize_as_string" + )] + pub probe_size: String, + #[serde( + default = "default_metadata_ffprobe_live_analyze_duration", + skip_serializing_if = "is_default_metadata_ffprobe_live_analyze_duration", + deserialize_with = "deserialize_as_string" + )] + pub live_analyze_duration: String, + #[serde( + default = "default_metadata_ffprobe_live_probe_size", + skip_serializing_if = "is_default_metadata_ffprobe_live_probe_size", + deserialize_with = "deserialize_as_string" + )] + pub live_probe_size: String, +} + +impl Default for FfprobeConfigDto { + fn default() -> Self { + Self { + enabled: false, + timeout: None, + analyze_duration: default_metadata_ffprobe_analyze_duration(), + probe_size: default_metadata_ffprobe_probe_size(), + live_analyze_duration: default_metadata_ffprobe_live_analyze_duration(), + live_probe_size: default_metadata_ffprobe_live_probe_size(), + } + } +} + +impl FfprobeConfigDto { + pub fn is_empty(&self) -> bool { + !self.enabled + && self.timeout.is_none() + && self.analyze_duration == default_metadata_ffprobe_analyze_duration() + && self.probe_size == default_metadata_ffprobe_probe_size() + && self.live_analyze_duration == default_metadata_ffprobe_live_analyze_duration() + && self.live_probe_size == default_metadata_ffprobe_live_probe_size() + } + + fn clean(&mut self) { + if self.timeout.is_some_and(|v| v == DEFAULT_FFPROBE_TIMEOUT_SECS) { + self.timeout = None; } } + fn prepare(&mut self) -> Result<(), TuliproxError> { + self.timeout = self.timeout.map(|timeout| timeout.max(MIN_DURATION_SECS)); + + let analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( + &self.analyze_duration, + MIN_DURATION_SECS, + "ffprobe.analyze_duration", + )?; + self.analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(analyze_duration_secs); + + let probe_size_bytes = parse_size_base_2(&self.probe_size) + .map_err(|err| crate::error::info_err!("Invalid size for `ffprobe.probe_size`: {err}"))? + .max(1); + self.probe_size = MetadataUpdateConfigDto::canonicalize_size_bytes(probe_size_bytes); + + let live_analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( + &self.live_analyze_duration, + MIN_DURATION_SECS, + "ffprobe.live_analyze_duration", + )?; + self.live_analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(live_analyze_duration_secs); + + let live_probe_size_bytes = parse_size_base_2(&self.live_probe_size) + .map_err(|err| crate::error::info_err!("Invalid size for `ffprobe.live_probe_size`: {err}"))? + .max(1); + self.live_probe_size = MetadataUpdateConfigDto::canonicalize_size_bytes(live_probe_size_bytes); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TmdbConfigDto { + #[serde(default)] + pub enabled: bool, + #[serde(default = "default_tmdb_api_key", skip_serializing_if = "is_tmdb_default_api_key")] + pub api_key: Option, + #[serde(default = "default_tmdb_rate_limit_ms", skip_serializing_if = "is_default_tmdb_rate_limit_ms")] + pub rate_limit_ms: u64, + #[serde(default = "default_tmdb_cache_duration_days", skip_serializing_if = "is_default_tmdb_cache_duration_days")] + pub cache_duration_days: u32, + #[serde(default = "default_tmdb_language", skip_serializing_if = "is_default_tmdb_language")] + pub language: String, + #[serde(default = "default_metadata_tmdb_cooldown", skip_serializing_if = "is_default_metadata_tmdb_cooldown")] + pub cooldown: String, +} + +impl Default for TmdbConfigDto { + fn default() -> Self { + Self { + enabled: false, + api_key: default_tmdb_api_key(), + rate_limit_ms: default_tmdb_rate_limit_ms(), + cache_duration_days: default_tmdb_cache_duration_days(), + language: default_tmdb_language(), + cooldown: default_metadata_tmdb_cooldown(), + } + } +} + +impl TmdbConfigDto { + pub fn is_empty(&self) -> bool { + !self.enabled + && self.api_key.as_ref().is_none_or(|api_key| api_key == TMDB_API_KEY) + && self.rate_limit_ms == default_tmdb_rate_limit_ms() + && self.cache_duration_days == default_tmdb_cache_duration_days() + && self.language == default_tmdb_language() + && self.cooldown == default_metadata_tmdb_cooldown() + } + + fn clean(&mut self) { + self.api_key = self.api_key.take().and_then(|api_key| { + let trimmed = api_key.trim(); + if trimmed.is_empty() || trimmed == TMDB_API_KEY { + None + } else { + Some(trimmed.to_string()) + } + }); + } + + fn prepare(&mut self) -> Result<(), TuliproxError> { + let cooldown_secs = + MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "tmdb.cooldown")?; + self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); + Ok(()) + } +} + +impl MetadataUpdateConfigDto { + pub fn is_empty(&self) -> bool { + self.log.is_empty() + && self.resolve.is_empty() + && self.probe.is_empty() + && self.ffprobe.is_empty() + && self.tmdb.is_empty() + && self.retry_delay == default_metadata_retry_delay() + && self.worker_idle_timeout == default_metadata_worker_idle_timeout() + && self.max_queue_size == default_metadata_max_queue_size() + } + + pub fn clean(&mut self) { + self.ffprobe.clean(); + self.tmdb.clean(); + } + pub fn prepare(&mut self) -> Result<(), TuliproxError> { - let queue_log_interval_secs = - Self::parse_and_clamp_duration(&self.queue_log_interval, MIN_DURATION_SECS, "queue_log_interval")?; - self.queue_log_interval = Self::canonicalize_seconds(queue_log_interval_secs); - - let progress_log_interval_secs = - Self::parse_and_clamp_duration(&self.progress_log_interval, MIN_DURATION_SECS, "progress_log_interval")?; - self.progress_log_interval = Self::canonicalize_seconds(progress_log_interval_secs); - - let max_resolve_retry_backoff_secs = Self::parse_and_clamp_duration( - &self.max_resolve_retry_backoff, - MIN_DURATION_SECS, - "max_resolve_retry_backoff", - )?; - self.max_resolve_retry_backoff = Self::canonicalize_seconds(max_resolve_retry_backoff_secs); - - let resolve_min_retry_base_secs = - Self::parse_and_clamp_duration(&self.resolve_min_retry_base, MIN_DURATION_SECS, "resolve_min_retry_base")?; - self.resolve_min_retry_base = Self::canonicalize_seconds(resolve_min_retry_base_secs); - - let resolve_exhaustion_reset_gap_secs = Self::parse_and_clamp_duration( - &self.resolve_exhaustion_reset_gap, - MIN_DURATION_SECS, - "resolve_exhaustion_reset_gap", - )?; - self.resolve_exhaustion_reset_gap = Self::canonicalize_seconds(resolve_exhaustion_reset_gap_secs); - - let probe_cooldown_secs = - Self::parse_and_clamp_duration(&self.probe_cooldown, MIN_DURATION_SECS, "probe_cooldown")?; - self.probe_cooldown = Self::canonicalize_seconds(probe_cooldown_secs); + self.log.prepare()?; + self.resolve.prepare()?; + self.probe.prepare()?; + self.ffprobe.prepare()?; + self.tmdb.prepare()?; let retry_delay_secs = Self::parse_and_clamp_duration(&self.retry_delay, MIN_DURATION_SECS, "retry_delay")?; self.retry_delay = Self::canonicalize_seconds(retry_delay_secs); - let probe_retry_load_retry_delay_secs = Self::parse_and_clamp_duration( - &self.probe_retry_load_retry_delay, - MIN_DURATION_SECS, - "probe_retry_load_retry_delay", - )?; - self.probe_retry_load_retry_delay = Self::canonicalize_seconds(probe_retry_load_retry_delay_secs); - let worker_idle_timeout_secs = Self::parse_and_clamp_duration(&self.worker_idle_timeout, MIN_DURATION_SECS, "worker_idle_timeout")?; self.worker_idle_timeout = Self::canonicalize_seconds(worker_idle_timeout_secs); - let probe_retry_backoff_step_1_secs = Self::parse_and_clamp_duration( - &self.probe_retry_backoff_step_1, - MIN_DURATION_SECS, - "probe_retry_backoff_step_1", - )?; - self.probe_retry_backoff_step_1 = Self::canonicalize_seconds(probe_retry_backoff_step_1_secs); - - let probe_retry_backoff_step_2_secs = Self::parse_and_clamp_duration( - &self.probe_retry_backoff_step_2, - MIN_DURATION_SECS, - "probe_retry_backoff_step_2", - )?; - self.probe_retry_backoff_step_2 = Self::canonicalize_seconds(probe_retry_backoff_step_2_secs); - - let probe_retry_backoff_step_3_secs = Self::parse_and_clamp_duration( - &self.probe_retry_backoff_step_3, - MIN_DURATION_SECS, - "probe_retry_backoff_step_3", - )?; - self.probe_retry_backoff_step_3 = Self::canonicalize_seconds(probe_retry_backoff_step_3_secs); - - self.max_attempts_resolve = self.max_attempts_resolve.max(MIN_ATTEMPTS); - self.max_attempts_probe = self.max_attempts_probe.max(MIN_ATTEMPTS); - self.backoff_jitter_percent = self.backoff_jitter_percent.min(MAX_JITTER_PERCENT); self.max_queue_size = self.max_queue_size.max(MIN_QUEUE_SIZE); - self.ffprobe_timeout = self.ffprobe_timeout.map(|timeout| timeout.max(MIN_DURATION_SECS)); - - let ffprobe_analyze_duration_secs = Self::parse_and_clamp_duration_with_required_unit( - &self.ffprobe_analyze_duration, - MIN_DURATION_SECS, - "ffprobe_analyze_duration", - )?; - self.ffprobe_analyze_duration = Self::canonicalize_seconds(ffprobe_analyze_duration_secs); - - let ffprobe_probe_size_bytes = parse_size_base_2(&self.ffprobe_probe_size) - .map_err(|err| crate::error::info_err!("Invalid size for `ffprobe_probe_size`: {err}"))? - .max(1); - self.ffprobe_probe_size = Self::canonicalize_size_bytes(ffprobe_probe_size_bytes); - - let ffprobe_live_analyze_duration_secs = Self::parse_and_clamp_duration_with_required_unit( - &self.ffprobe_live_analyze_duration, - MIN_DURATION_SECS, - "ffprobe_live_analyze_duration", - )?; - self.ffprobe_live_analyze_duration = Self::canonicalize_seconds(ffprobe_live_analyze_duration_secs); - - let ffprobe_live_probe_size_bytes = parse_size_base_2(&self.ffprobe_live_probe_size) - .map_err(|err| crate::error::info_err!("Invalid size for `ffprobe_live_probe_size`: {err}"))? - .max(1); - self.ffprobe_live_probe_size = Self::canonicalize_size_bytes(ffprobe_live_probe_size_bytes); self.clean(); @@ -346,50 +552,55 @@ mod tests { assert!(cfg.is_empty()); } + #[test] + fn prepare_keeps_default_config_empty() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.prepare().expect("metadata update config defaults should be valid"); + assert!(cfg.is_empty()); + } + #[test] fn prepare_parses_duration_suffixes() { - let mut cfg = MetadataUpdateConfigDto { - queue_log_interval: "1m".to_string(), - progress_log_interval: "2h".to_string(), - probe_cooldown: "1d".to_string(), - ..MetadataUpdateConfigDto::default() - }; + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.log.queue_interval = "1m".to_string(); + cfg.log.progress_interval = "2h".to_string(); + cfg.probe.cooldown = "1d".to_string(); + cfg.tmdb.cooldown = "2d".to_string(); cfg.prepare().expect("metadata update config should parse duration values"); - assert_eq!(cfg.queue_log_interval, "1m"); - assert_eq!(cfg.progress_log_interval, "2h"); - assert_eq!(cfg.probe_cooldown, "1d"); + assert_eq!(cfg.log.queue_interval, "1m"); + assert_eq!(cfg.log.progress_interval, "2h"); + assert_eq!(cfg.probe.cooldown, "1d"); + assert_eq!(cfg.tmdb.cooldown, "2d"); } #[test] fn prepare_clamps_minimum_values() { - let mut cfg = MetadataUpdateConfigDto { - queue_log_interval: "0".to_string(), - max_attempts_resolve: 0, - max_attempts_probe: 0, - max_queue_size: 0, - ffprobe_timeout: Some(0), - ffprobe_analyze_duration: "0s".to_string(), - ffprobe_probe_size: "0".to_string(), - ..MetadataUpdateConfigDto::default() - }; + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.log.queue_interval = "0".to_string(); + cfg.resolve.max_attempts = 0; + cfg.probe.max_attempts = 0; + cfg.max_queue_size = 0; + cfg.ffprobe.timeout = Some(0); + cfg.ffprobe.analyze_duration = "0s".to_string(); + cfg.ffprobe.probe_size = "0".to_string(); cfg.prepare().expect("metadata update config should clamp minimum values"); - assert_eq!(cfg.queue_log_interval, "1s"); - assert_eq!(cfg.max_attempts_resolve, 1); - assert_eq!(cfg.max_attempts_probe, 1); + assert_eq!(cfg.log.queue_interval, "1s"); + assert_eq!(cfg.resolve.max_attempts, 1); + assert_eq!(cfg.probe.max_attempts, 1); assert_eq!(cfg.max_queue_size, 1); - assert_eq!(cfg.ffprobe_timeout, Some(1)); - assert_eq!(cfg.ffprobe_analyze_duration, "1s"); - assert_eq!(cfg.ffprobe_probe_size, "1B"); + assert_eq!(cfg.ffprobe.timeout, Some(1)); + assert_eq!(cfg.ffprobe.analyze_duration, "1s"); + assert_eq!(cfg.ffprobe.probe_size, "1B"); } #[test] fn prepare_rejects_invalid_duration_unit() { - let mut cfg = - MetadataUpdateConfigDto { queue_log_interval: "1w".to_string(), ..MetadataUpdateConfigDto::default() }; + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.log.queue_interval = "1w".to_string(); let result = cfg.prepare(); assert!(result.is_err(), "invalid duration unit must fail"); @@ -397,38 +608,36 @@ mod tests { #[test] fn prepare_canonicalizes_to_larger_units() { - let mut cfg = MetadataUpdateConfigDto { - probe_cooldown: "604800".to_string(), - worker_idle_timeout: "60".to_string(), - probe_retry_backoff_step_3: "3600".to_string(), - ffprobe_analyze_duration: "10s".to_string(), - ffprobe_probe_size: "10485760".to_string(), - ffprobe_live_analyze_duration: "5s".to_string(), - ffprobe_live_probe_size: "5242880".to_string(), - ..MetadataUpdateConfigDto::default() - }; + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.probe.cooldown = "604800".to_string(); + cfg.tmdb.cooldown = "259200".to_string(); + cfg.worker_idle_timeout = "60".to_string(); + cfg.probe.retry_backoff_step_3 = "3600".to_string(); + cfg.ffprobe.analyze_duration = "10s".to_string(); + cfg.ffprobe.probe_size = "10485760".to_string(); + cfg.ffprobe.live_analyze_duration = "5s".to_string(); + cfg.ffprobe.live_probe_size = "5242880".to_string(); cfg.prepare().expect("metadata update config should canonicalize durations"); - assert_eq!(cfg.probe_cooldown, "7d"); + assert_eq!(cfg.probe.cooldown, "7d"); + assert_eq!(cfg.tmdb.cooldown, "3d"); assert_eq!(cfg.worker_idle_timeout, "1m"); - assert_eq!(cfg.probe_retry_backoff_step_3, "1h"); - assert_eq!(cfg.ffprobe_analyze_duration, "10s"); - assert_eq!(cfg.ffprobe_probe_size, "10MB"); - assert_eq!(cfg.ffprobe_live_analyze_duration, "5s"); - assert_eq!(cfg.ffprobe_live_probe_size, "5MB"); + assert_eq!(cfg.probe.retry_backoff_step_3, "1h"); + assert_eq!(cfg.ffprobe.analyze_duration, "10s"); + assert_eq!(cfg.ffprobe.probe_size, "10MB"); + assert_eq!(cfg.ffprobe.live_analyze_duration, "5s"); + assert_eq!(cfg.ffprobe.live_probe_size, "5MB"); } #[test] fn prepare_rejects_ffprobe_duration_without_unit() { - let mut cfg = MetadataUpdateConfigDto { - ffprobe_analyze_duration: "10000000".to_string(), - ..MetadataUpdateConfigDto::default() - }; + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.ffprobe.analyze_duration = "10000000".to_string(); let result = cfg.prepare(); assert!(result.is_err(), "numeric ffprobe analyze duration without unit must fail"); - let err_text = result.unwrap_err().to_string(); - assert!(err_text.contains("ffprobe_analyze_duration")); + let err_text = result.expect_err("validation should fail").to_string(); + assert!(err_text.contains("ffprobe.analyze_duration")); } } diff --git a/shared/src/utils/default_utils.rs b/shared/src/utils/default_utils.rs index 1b30d7258..269d98517 100644 --- a/shared/src/utils/default_utils.rs +++ b/shared/src/utils/default_utils.rs @@ -141,6 +141,8 @@ pub fn is_default_metadata_resolve_exhaustion_reset_gap(v: &String) -> bool { } pub fn default_metadata_probe_cooldown() -> String { "7d".to_string() } pub fn is_default_metadata_probe_cooldown(v: &String) -> bool { *v == default_metadata_probe_cooldown() } +pub fn default_metadata_tmdb_cooldown() -> String { "7d".to_string() } +pub fn is_default_metadata_tmdb_cooldown(v: &String) -> bool { *v == default_metadata_tmdb_cooldown() } pub fn default_metadata_retry_delay() -> String { "2s".to_string() } pub fn is_default_metadata_retry_delay(v: &String) -> bool { *v == default_metadata_retry_delay() } pub fn default_metadata_probe_retry_load_retry_delay() -> String { "1m".to_string() }