diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b79c279..769f0c054 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ - **Sort**: Sort can now use filter to sort specific entries. ```yaml + sort: match_as_ascii: true rules: @@ -136,7 +137,10 @@ active URL of the specified provider. - **config.yml**: - Added `metadata_update` (optional) with grouped sections: `log`, `resolve`, `probe`, `ffprobe`, `tmdb`. - - Added `metadata_update.cache_path` (default `metadata`): shared storage directory for TMDB cache and metadata files (moved from `library.metadata.path`). + - Added `metadata_update.cache_path` (default `metadata`): shared storage directory for TMDB cache and metadata files + (moved from `library.metadata.path`). + - Added `metadata_update.no_change_cache_ttl_secs` (default `3600`): TTL in seconds for the no-change + deduplication cache used by background metadata resolve tasks. - Added `metadata_update.tmdb.cooldown` (default `7d`) for successful TMDB no-match cooldown behavior. - Added `metadata_update.ffprobe.enabled` (default: false), `metadata_update.ffprobe.timeout`, and ffprobe probe/analyze size settings. - `metadata_update.ffprobe.analyze_duration` and `metadata_update.ffprobe.live_analyze_duration` require explicit unit suffixes (`s|m|h|d`). @@ -146,7 +150,6 @@ active URL of the specified provider. - Added `resolve_tmdb`: Triggers TMDB lookup if ID is missing. - Added `probe_stream`: Triggers ffprobe if technical info is missing. - Added `probe_delay`: Delay between probe tasks (default `50` seconds). - - Added `disable_hls_streaming`: Disables HLS reverse-proxy mode for live streams and requests direct TS stream endpoints instead. - Added `staged.enabled`: Disables/enables the staged input - **source.yml (target output)**: - Added `probe_live`: Enables background probing for Live TV streams (default disabled). @@ -234,7 +237,6 @@ active URL of the specified provider. lookups. - **Client Requests**: Extended debug logging for client requests and ID chain. - **XTream Fixes**: Fixed series/catch-up lookups using `series-info virtual_id`. -- **Catchup Stability**: Fixed catchup parent mapping/session isolation to keep playback/account selection stable on the same user account. - **Cloudflare Header**: Added `cloudflare_header` to reverse proxy `disable_header` settings. - **Kick Seconds**: `kick_secs` added to `config.yml web_ui` config. - **Improved connection handling** for users with strict connection limits during streaming operations. diff --git a/Cargo.lock b/Cargo.lock index 3d83193b5..c28892550 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1479,12 +1479,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasip2", "wasip3", - "wasm-bindgen", ] [[package]] diff --git a/README.md b/README.md index b089954e6..9e45041fd 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,7 @@ metadata_update: cache_duration_days: 0 language: en-US cooldown: 7d + match_threshold: 86 ffprobe: enabled: true timeout: 60 @@ -469,6 +470,7 @@ metadata_update: retry_delay: 2s worker_idle_timeout: 1m max_queue_size: 100000 + no_change_cache_ttl_secs: 3600 ``` **Field Reference (`metadata_update`):** @@ -498,9 +500,12 @@ metadata_update: - `probe.backoff_jitter_percent` (default `20`): Random jitter percentage applied to resolve/probe retry backoff to avoid synchronized retries. - `tmdb.cooldown` (default `7d`): Cooldown duration after a TMDB lookup completed successfully but returned no match. - `tmdb.enabled` / `tmdb.api_key` / `tmdb.rate_limit_ms` / `tmdb.cache_duration_days` / `tmdb.language`: TMDB resolver settings. +- `tmdb.match_threshold` (default `86`): TMDB match threshold for search results for TMDB ID resolution. - `retry_delay` (default `2s`): Minimum retry delay for transient worker errors (for example temporary connection/resource issues). - `worker_idle_timeout` (default `1m`): Idle timeout for metadata worker shutdown when there is no immediate work. - `max_queue_size` (default `100000`): Maximum pending metadata tasks per input before new tasks are rejected. +- `no_change_cache_ttl_secs` (default `3600`): TTL in seconds for the per-item no-change deduplication cache. + While valid, identical reason sets are skipped. - `ffprobe.enabled` (default `false`): Globally enables/disables FFprobe-based stream analysis. - `ffprobe.timeout` (default `60`): FFprobe process timeout in seconds. - `ffprobe.analyze_duration` (default `10s`): FFprobe `-analyzeduration` @@ -517,6 +522,7 @@ metadata_update: - `retry_delay`: Minimum waiting time before a task is retried after a temporary problem. Prevents very fast retry loops. - `worker_idle_timeout`: How long a metadata worker stays alive with no new work before it can stop and free resources. - `max_queue_size`: Safety limit for how many metadata tasks are kept in memory at once. +- `no_change_cache_ttl_secs`: How long (seconds) a no-change resolve result is remembered before the same reason set is allowed to execute again. - `log.queue_interval`: How often Tuliprox writes queue status logs. - `log.progress_interval`: How often Tuliprox writes progress logs while tasks are running. - `resolve.max_retry_backoff`: Longest wait time between repeated resolve retries. diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index f7efcf8e9..bd8478a6e 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -1710,6 +1710,7 @@ pub fn create_api_proxy_user(app_state: &Arc) -> ProxyUserCredentials status: None, ui_enabled: false, comment: None, + t_is_api_user: true, } } diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index 1800a3bda..1c98e3a1b 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -1,6 +1,7 @@ +use crate::api::model::AppState; use crate::{ api::api_utils::{empty_json_list_response, json_or_bin_response, stream_json_or_bin_response_stream}, - model::{AppConfig, ConfigInput, ConfigTarget}, + model::{ConfigInput, ConfigTarget}, repository::{ iter_raw_m3u_input_playlist, iter_raw_m3u_target_playlist, iter_raw_xtream_input_playlist, iter_raw_xtream_target_playlist, @@ -10,6 +11,7 @@ use crate::{ use axum::response::IntoResponse; use log::warn; use serde_json::json; +use shared::utils::{concat_path, concat_path_leading_slash, obfuscate_text, Internable}; use shared::{ model::{ InputType, M3uPlaylistItem, PlaylistItemType, TargetType, UiPlaylistItem, XtreamCluster, XtreamPlaylistItem, @@ -21,13 +23,21 @@ use tokio_stream::StreamExt; pub(in crate::api::endpoints) async fn get_playlist_for_target( cfg_target: Option<&ConfigTarget>, - cfg: &AppConfig, + app_state: &Arc, cluster: XtreamCluster, accept: Option<&str>, ) -> impl IntoResponse + Send { + let config = app_state.app_config.config.load(); + let web_ui_path = config + .web_ui + .as_ref() + .and_then(|w| w.path.as_ref()) + .map_or("", String::as_str); + let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); + let encrypt_secret = app_state.get_encrypt_secret(); if let Some(target) = cfg_target { if target.has_output(TargetType::Xtream) { - let Some(channel_iterator) = iter_raw_xtream_target_playlist(cfg, target, cluster).await else { + let Some(channel_iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, target, cluster).await else { return empty_json_list_response(); }; let item_filter = if cluster == XtreamCluster::Series { @@ -37,10 +47,10 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target( } else { |_pli: &XtreamPlaylistItem| true }; - let converted_stream = channel_iterator.filter(item_filter).map(UiPlaylistItem::from); + let converted_stream = channel_iterator.filter(item_filter).map(UiPlaylistItem::from).map(move |uiu| rewrite_resource_url(&encrypt_secret, &resource_url, uiu)); return stream_json_or_bin_response_stream(accept, converted_stream).into_response(); } else if target.has_output(TargetType::M3u) { - let Some(channel_iterator) = iter_raw_m3u_target_playlist(cfg, target, Some(cluster)).await else { + let Some(channel_iterator) = iter_raw_m3u_target_playlist(&app_state.app_config, target, Some(cluster)).await else { return empty_json_list_response(); }; let item_filter = if cluster == XtreamCluster::Series { @@ -54,7 +64,7 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target( let converted_stream = channel_iterator.filter_map(move |res| match res { Ok(pli) => { if item_filter(&pli) { - Some(UiPlaylistItem::from(pli)) + Some(rewrite_resource_url(&encrypt_secret, &resource_url, UiPlaylistItem::from(pli))) } else { None } @@ -70,21 +80,30 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target( (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response() } +fn rewrite_resource_url(encrypt_secret: &[u8; 16], resource_url: &str, item: UiPlaylistItem) -> UiPlaylistItem { + if item.logo.is_empty() { + return item; + } + let mut item = item; + item.logo = concat_path(resource_url, &obfuscate_text(encrypt_secret, &item.logo)).intern(); + item +} + pub(in crate::api::endpoints) async fn get_playlist_for_input( cfg_input: Option<&Arc>, - cfg: &AppConfig, + app_state: &Arc, cluster: XtreamCluster, accept: Option<&str>, ) -> impl IntoResponse + Send { if let Some(input) = cfg_input { if matches!(input.input_type, InputType::Xtream | InputType::XtreamBatch) { - let Some(channel_iterator) = iter_raw_xtream_input_playlist(cfg, input, cluster).await else { + let Some(channel_iterator) = iter_raw_xtream_input_playlist(&app_state.app_config, input, cluster).await else { return empty_json_list_response(); }; let converted_stream = channel_iterator.map(UiPlaylistItem::from); return stream_json_or_bin_response_stream(accept, converted_stream).into_response(); } else if matches!(input.input_type, InputType::M3u | InputType::M3uBatch) { - let Some(channels) = iter_raw_m3u_input_playlist(cfg, input, Some(cluster)).await else { + let Some(channels) = iter_raw_m3u_input_playlist(&app_state.app_config, input, Some(cluster)).await else { return empty_json_list_response(); }; let converted_stream = channels.filter_map(|res| match res { @@ -103,20 +122,20 @@ pub(in crate::api::endpoints) async fn get_playlist_for_input( pub(in crate::api::endpoints) async fn get_playlist_for_custom_provider( client: &reqwest::Client, cfg_input: Option<&Arc>, - app_config: &Arc, + app_state: &Arc, cluster: XtreamCluster, accept: Option<&str>, ) -> impl IntoResponse + Send { - let cfg = app_config.config.load(); + let cfg = app_state.app_config.config.load(); match cfg_input { Some(input) => { let (result, errors) = match input.input_type { InputType::M3u | InputType::M3uBatch => { - m3u::download_m3u_playlist(app_config, client, &cfg, input).await + m3u::download_m3u_playlist(&app_state.app_config, client, &cfg, input).await } InputType::Xtream | InputType::XtreamBatch => { let (pl, err, _) = - xtream::download_xtream_playlist(app_config, client, input, Some(&[cluster])).await; + xtream::download_xtream_playlist(&app_state.app_config, client, input, Some(&[cluster])).await; (pl, err) } InputType::Library => { diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 7d968c7f8..e66f1f0ea 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -204,9 +204,11 @@ pub(in crate::api) async fn handle_hls_stream_request( }; match download_result { Ok((content, response_url)) => { + let encrypt_secret = app_state.get_encrypt_secret(); + let base_url = server_info.get_base_url(); let rewrite_hls_props = RewriteHlsProps { - secret: &app_state.app_config.encrypt_secret, - base_url: &server_info.get_base_url(), + secret: &encrypt_secret, + base_url: &base_url, content: &content, hls_url: response_url, virtual_id, @@ -337,11 +339,12 @@ async fn hls_api_stream( .into_response(); } - let hls_url = - match get_hls_session_token_and_url_from_token(&app_state.app_config.encrypt_secret, ¶ms.token) { - Some((Some(session_token), hls_url)) if session.token.eq(&session_token) => hls_url, - _ => return axum::http::StatusCode::BAD_REQUEST.into_response(), - }; + let encrypt_secret = app_state.get_encrypt_secret(); + let hls_url = match get_hls_session_token_and_url_from_token(&encrypt_secret, ¶ms.token) { + Some((Some(session_token), hls_url)) if session.token.eq(&session_token) => hls_url, + Some((None, hls_url)) => hls_url, + _ => return axum::http::StatusCode::BAD_REQUEST.into_response(), + }; let hls_url = hls_url.intern(); session.stream_url = hls_url.clone(); if session.virtual_id == virtual_id { diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index 19a0e4230..d726e7fd2 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -4,7 +4,8 @@ use crate::{ endpoints::{ download_api, extract_accept_header::ExtractAcceptHeader, library_api::library_api_register, user_api::user_api_register, v1_api_config::v1_api_config_register, - v1_api_playlist::v1_api_playlist_register, v1_api_user::v1_api_user_register, + v1_api_playlist::{v1_api_playlist_register_protected, v1_api_playlist_register_public}, + v1_api_user::v1_api_user_register, }, model::AppState, }, @@ -162,20 +163,23 @@ pub fn v1_api_register( app_state: Arc, web_ui_path: &str, ) -> axum::Router> { - let mut router = axum::Router::new(); - router = router + let public_router = v1_api_playlist_register_public(axum::Router::new()); + + let mut protected_router = axum::Router::new(); + protected_router = protected_router .route("/status", axum::routing::get(status)) .route("/streams", axum::routing::get(streams)) .route("/geoip/update", axum::routing::get(geoip_update)) .route("/file/download", axum::routing::post(download_api::queue_download_file)) .route("/file/download/info", axum::routing::get(download_api::download_file_info)) .route("/ipinfo", axum::routing::get(ipinfo)); - router = v1_api_config_register(router); - router = v1_api_user_register(router); - router = v1_api_playlist_register(router); - router = library_api_register(router); + protected_router = v1_api_config_register(protected_router); + protected_router = v1_api_user_register(protected_router); + protected_router = v1_api_playlist_register_protected(protected_router); + protected_router = library_api_register(protected_router); if web_auth_enabled { - router = router.route_layer(axum::middleware::from_fn_with_state(Arc::clone(&app_state), validator_admin)); + protected_router = + protected_router.route_layer(axum::middleware::from_fn_with_state(Arc::clone(&app_state), validator_admin)); } let config = app_state.app_config.config.load(); @@ -183,5 +187,7 @@ pub fn v1_api_register( if config.web_ui.as_ref().is_none_or(|c| c.user_ui_enabled) { base_router = base_router.merge(user_api_register(app_state, web_ui_path)); } - base_router.nest(&concat_path_leading_slash(web_ui_path, "api/v1"), router) + base_router + .nest(&concat_path_leading_slash(web_ui_path, "api/v1"), public_router) + .nest(&concat_path_leading_slash(web_ui_path, "api/v1"), protected_router) } diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index 76d828899..627be92a2 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -26,6 +26,8 @@ use shared::{ }; use std::sync::Arc; use url::Url; +use shared::utils::deobfuscate_text; +use crate::api::api_utils::resource_response; fn create_config_input_for_m3u(url: &str) -> ConfigInput { ConfigInput { @@ -120,12 +122,11 @@ async fn playlist_content( playlist_req: &PlaylistRequest, cluster: XtreamCluster, ) -> impl IntoResponse + Send { - let _config = app_state.app_config.config.load(); let client = app_state.http_client.load(); match playlist_req { PlaylistRequest::Target(target_id) => get_playlist_for_target( app_state.app_config.get_target_by_id(*target_id).as_deref(), - &app_state.app_config, + app_state, cluster, accept.as_deref(), ) @@ -133,7 +134,7 @@ async fn playlist_content( .into_response(), PlaylistRequest::Input(input_id) => get_playlist_for_input( app_state.app_config.get_input_by_id(*input_id).as_ref(), - &app_state.app_config, + app_state, cluster, accept.as_deref(), ) @@ -145,7 +146,7 @@ async fn playlist_content( get_playlist_for_custom_provider( client.as_ref(), Some(&input), - &app_state.app_config, + app_state, cluster, accept.as_deref(), ) @@ -164,7 +165,7 @@ async fn playlist_content( get_playlist_for_custom_provider( client.as_ref(), Some(&input), - &app_state.app_config, + app_state, cluster, accept.as_deref(), ) @@ -293,7 +294,20 @@ async fn playlist_epg( axum::http::StatusCode::NO_CONTENT.into_response() } -pub fn v1_api_playlist_register(router: Router>) -> axum::Router> { +async fn playlist_resource( + req_headers: axum::http::HeaderMap, + axum::extract::Path(resource): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + let encrypt_secret = app_state.get_encrypt_secret(); + if let Ok(resource_url) = deobfuscate_text(&encrypt_secret, &resource) { + resource_response(&app_state, &resource_url, &req_headers, None).await.into_response() + } else { + axum::http::StatusCode::BAD_REQUEST.into_response() + } +} + +pub fn v1_api_playlist_register_protected(router: Router>) -> axum::Router> { router .route("/playlist/webplayer", axum::routing::post(playlist_webplayer)) .route("/playlist/update", axum::routing::post(playlist_update)) @@ -305,6 +319,12 @@ pub fn v1_api_playlist_register(router: Router>) -> axum::Router>, +) -> axum::Router> { + router.route("/playlist/resource/{resource}", axum::routing::get(playlist_resource)) +} + async fn playlist_episode_item( axum::extract::Path(virtual_id): axum::extract::Path, axum::extract::State(app_state): axum::extract::State>, diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index 2d2748ee4..9c0db035c 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -494,8 +494,7 @@ async fn epg_api_resource( return axum::http::StatusCode::FORBIDDEN.into_response(); } - let encrypt_secret = - app_state.app_config.get_reverse_proxy_rewrite_secret().unwrap_or_else(|| app_state.app_config.encrypt_secret); + let encrypt_secret = app_state.get_encrypt_secret(); if let Ok(resource_url) = deobscure_text(&encrypt_secret, &resource) { resource_response(&app_state, &resource_url, &req_headers, None).await.into_response() } else { diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index 9feac8f5c..e12c22f38 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -789,13 +789,14 @@ pub async fn xtream_get_stream_info_response( return try_unwrap_body!(empty_json_response_as_array()); }; - let server_info = app_state.app_config.get_user_server_info(user); + let encrypt_secret = app_state.get_encrypt_secret(); + let options = xtream_mapping_option_from_target_options( target, xtream_output, &app_state.app_config, user, - Some(server_info.get_base_url().as_str()), + encrypt_secret, ); return axum::Json(pli.to_info_document(&options)).into_response(); } @@ -1209,17 +1210,17 @@ async fn xtream_player_api(api_req: UserApiRequest, app_state: &Arc) - let result = match action { crate::model::XC_ACTION_GET_LIVE_STREAMS => skip_flag_optional!( skip_live, - xtream_load_rewrite_playlist(XtreamCluster::Live, &app_state.app_config, &target, category_id, &user) + xtream_load_rewrite_playlist(XtreamCluster::Live, app_state, &target, category_id, &user) .await ), crate::model::XC_ACTION_GET_VOD_STREAMS => skip_flag_optional!( skip_vod, - xtream_load_rewrite_playlist(XtreamCluster::Video, &app_state.app_config, &target, category_id, &user) + xtream_load_rewrite_playlist(XtreamCluster::Video, app_state, &target, category_id, &user) .await ), crate::model::XC_ACTION_GET_SERIES => skip_flag_optional!( skip_series, - xtream_load_rewrite_playlist(XtreamCluster::Series, &app_state.app_config, &target, category_id, &user) + xtream_load_rewrite_playlist(XtreamCluster::Series, app_state, &target, category_id, &user) .await ), _ => Some(info_err_res!("Unknown api call: {action} for target: {}", &target.name)), diff --git a/backend/src/api/model/app_state.rs b/backend/src/api/model/app_state.rs index f21a14ccf..ad4a06f17 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/src/api/model/app_state.rs @@ -515,6 +515,12 @@ impl AppState { config.proxy.as_ref().is_some_and(|proxy| should_use_manual_redirect_for_proxy(proxy.url.as_str())) || proxy_env_present() } + + pub fn get_encrypt_secret(&self) -> [u8;16] { + self.app_config + .get_reverse_proxy_rewrite_secret() + .unwrap_or(self.app_config.encrypt_secret) + } } fn proxy_env_present() -> bool { should_use_manual_redirects_for_env_vars(std::env::vars_os()) } diff --git a/backend/src/api/model/metadata_update_manager.rs b/backend/src/api/model/metadata_update_manager.rs index f834de836..ad7183703 100644 --- a/backend/src/api/model/metadata_update_manager.rs +++ b/backend/src/api/model/metadata_update_manager.rs @@ -31,7 +31,7 @@ use std::{ io, path::{Path, PathBuf}, sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, Arc, OnceLock, Weak, }, time::{Duration, Instant}, @@ -87,6 +87,7 @@ struct MetadataUpdateRuntimeSettings { metadata_retry_load_retry_delay_secs: i64, worker_idle_timeout_secs: u64, max_queue_size: usize, + no_change_cache_ttl_secs: u64, probe_retry_backoff_step_1_secs: u64, probe_retry_backoff_step_2_secs: u64, probe_retry_backoff_step_3_secs: u64, @@ -131,6 +132,7 @@ impl MetadataUpdateRuntimeSettings { metadata_retry_load_retry_delay_secs: to_i64(cfg.probe.retry_load_retry_delay_secs), worker_idle_timeout_secs: cfg.worker_idle_timeout_secs.max(1), max_queue_size: cfg.max_queue_size.max(1), + no_change_cache_ttl_secs: cfg.no_change_cache_ttl_secs.max(1), probe_retry_backoff_step_1_secs: cfg.probe.retry_backoff_step_1_secs.max(1), probe_retry_backoff_step_2_secs: cfg.probe.retry_backoff_step_2_secs.max(1), probe_retry_backoff_step_3_secs: cfg.probe.retry_backoff_step_3_secs.max(1), @@ -339,7 +341,7 @@ impl TaskRetryState { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -enum MetadataRetryDbKey { +pub enum MetadataRetryDbKey { VodId(u32), VodText(String), SeriesId(u32), @@ -409,7 +411,7 @@ impl RetryStateDbValue { } #[derive(Debug, Clone, Serialize, Deserialize)] -struct MetadataRetryDbValue { +pub struct MetadataRetryDbValue { resolve: Option, probe: Option, tmdb: Option, @@ -645,6 +647,7 @@ impl MetadataUpdateManager { metadata_retry_load_retry_at_ts: None, last_retry_state_prune_at_ts: None, scheduled_requeues: Arc::new(DashMap::new()), + recently_completed_no_change: HashMap::new(), }, )); @@ -900,6 +903,10 @@ struct InputWorker { // Shared with detached delayed requeue tasks spawned in `schedule_requeue_at`. // A plain HashMap cannot be moved safely into those `'static` tasks. scheduled_requeues: Arc>, + // Cache of tasks that recently completed with no changes (Ok(None)). + // Prevents repeated resolution of already-resolved items across playlist refreshes. + // Stores the reason set so that tasks with new/different reasons are not wrongly skipped. + recently_completed_no_change: HashMap, } #[derive(Debug, Clone, Copy)] @@ -991,6 +998,20 @@ impl InputWorker { skip_execution = true; } + // Skip tasks that recently completed with no changes (already resolved in DB). + // Only skip when the incoming reason set exactly matches the cached reason set. + if !skip_execution + && self.should_skip_recent_no_change_task(¤t_key, &task_for_execution, &runtime_settings) + { + debug_if_enabled!( + "Skipping recently-resolved no-change task for input {}: {}", + input_name, + task_for_execution + ); + remove_current_task = true; + skip_execution = true; + } + if !skip_execution { let mut clear_tmdb_state = false; if let Some(state_bundle) = self.retry_states.get(¤t_key) { @@ -1180,6 +1201,17 @@ impl InputWorker { self.resolve_exhausted.remove(¤t_key); self.scheduled_requeues.remove(¤t_key); + // Cache tasks that completed with no changes to skip redundant re-resolution. + if !task_outcome.task_changed + && Self::is_resolve_task(&task_for_execution) + && !task_outcome.tmdb_pending + { + let reasons = Self::task_reason(&task_for_execution); + self.recently_completed_no_change.insert(current_key.clone(), (Instant::now(), reasons)); + } else { + self.recently_completed_no_change.remove(¤t_key); + } + if last_progress_log_at.elapsed() >= runtime_settings.progress_log_interval { // current_key is removed from pending_tasks later in this loop iteration; // subtract it here so "remaining" reflects the post-success queue size. @@ -1762,6 +1794,10 @@ impl InputWorker { for key in stale_resolve_exhausted_keys { self.resolve_exhausted.remove(&key); } + + let no_change_ttl = Duration::from_secs(runtime_settings.no_change_cache_ttl_secs); + self.recently_completed_no_change + .retain(|_, (completed_at, _)| completed_at.elapsed() < no_change_ttl); } fn release_db_handles(&mut self) { @@ -1867,6 +1903,39 @@ impl InputWorker { matches!(task, UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. }) } + #[inline] + fn task_reason(task: &UpdateTask) -> ResolveReasonSet { + match task { + UpdateTask::ResolveVod { reason, .. } + | UpdateTask::ResolveSeries { reason, .. } + | UpdateTask::ProbeLive { reason, .. } + | UpdateTask::ProbeStream { reason, .. } => *reason, + } + } + + fn should_skip_recent_no_change_task( + &mut self, + current_key: &TaskKey, + task_for_execution: &UpdateTask, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) -> bool { + let Some((completed_at, cached_reasons)) = + self.recently_completed_no_change.get(current_key).copied() + else { + return false; + }; + + let ttl = Duration::from_secs(runtime_settings.no_change_cache_ttl_secs); + let current_reasons = Self::task_reason(task_for_execution); + if completed_at.elapsed() < ttl && current_reasons == cached_reasons { + self.scheduled_requeues.remove(current_key); + return true; + } + + self.recently_completed_no_change.remove(current_key); + false + } + #[inline] fn should_trigger_playlist_update_for_task(task: &UpdateTask, task_changed: bool) -> bool { task_changed && !Self::is_probe_task(task) && !Self::is_probe_only_resolve_task(task) @@ -2540,19 +2609,6 @@ impl InputWorker { None } - fn vod_tmdb_pending_for_reason(props: &VideoStreamProperties, reason: ResolveReasonSet) -> bool { - let tmdb_missing = reason.contains(ResolveReason::Tmdb) && props.tmdb.is_none(); - let date_missing = reason.contains(ResolveReason::Date) - && props.details.as_ref().and_then(|d| d.release_date.as_ref()).is_none(); - tmdb_missing || date_missing - } - - fn series_tmdb_pending_for_reason(props: &SeriesStreamProperties, reason: ResolveReasonSet) -> bool { - let tmdb_missing = reason.contains(ResolveReason::Tmdb) && props.tmdb.is_none(); - let date_missing = reason.contains(ResolveReason::Date) && props.release_date.is_none(); - tmdb_missing || date_missing - } - #[allow(clippy::too_many_lines)] async fn process_task_static( input_name: &Arc, @@ -2672,7 +2728,7 @@ impl InputWorker { app_state.connection_manager.release_provider_handle(provider_handle).await; } match res { - Ok(()) => { + Ok(tmdb_and_date_present) => { let task_changed = match task { UpdateTask::ResolveVod { .. } => collector.vod.len() > pre_vod_updates, UpdateTask::ResolveSeries { .. } => collector.series.len() > pre_series_updates, @@ -2680,22 +2736,9 @@ impl InputWorker { UpdateTask::ProbeStream { .. } => true, }; let tmdb_pending = match task { - UpdateTask::ResolveVod { reason, .. } => { + UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { if reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) { - collector - .vod - .get(pre_vod_updates) - .is_none_or(|(_, props)| Self::vod_tmdb_pending_for_reason(props, *reason)) - } else { - false - } - } - UpdateTask::ResolveSeries { reason, .. } => { - if reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) { - collector - .series - .get(pre_series_updates) - .is_none_or(|(_, props)| Self::series_tmdb_pending_for_reason(props, *reason)) + !tmdb_and_date_present } else { false } @@ -2713,9 +2756,10 @@ impl InputWorker { UpdateTask::ProbeLive { .. } => true, // Local library probing is fully local and must not depend on provider capacity. UpdateTask::ProbeStream { .. } => !matches!(input_type, InputType::Library), - UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { - reason.contains(ResolveReason::Probe) - } + // Resolve tasks handle their own probe connection acquisition internally. + // This avoids holding a provider connection for the entire duration of + // info fetch + TMDB resolve + probe, reducing "provider exhausted" errors. + UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. } => false, } } @@ -2730,7 +2774,9 @@ impl InputWorker { collector: &mut BatchResultCollector, db_handles: &mut HashMap, failed_clusters: &mut HashSet, - ) -> Result<(), TuliproxError> { + ) -> Result { + // The returned bool indicates whether both TMDB id and release date are already present in the DB + // (used by the caller to avoid false-positive "no match" cooldowns). match task { UpdateTask::ResolveVod { id, reason, .. } => { let fetch_info = reason.contains(ResolveReason::Info); @@ -2752,6 +2798,7 @@ impl InputWorker { .await }; + let tmdb_and_date_present = AtomicBool::new(false); match update_vod_metadata( &app_state.app_config, client, @@ -2765,14 +2812,15 @@ impl InputWorker { resolve_tmdb, will_probe, query_opt, + Some(&tmdb_and_date_present), ) .await { Ok(Some(props)) => { collector.add_vod(id.clone(), props); - Ok(()) + Ok(tmdb_and_date_present.load(Ordering::Relaxed)) } - Ok(None) => Ok(()), + Ok(None) => Ok(tmdb_and_date_present.load(Ordering::Relaxed)), Err(e) => Err(e), } } @@ -2797,6 +2845,7 @@ impl InputWorker { .await }; + let tmdb_and_date_present = AtomicBool::new(false); match update_series_metadata( &app_state.app_config, client, @@ -2811,14 +2860,15 @@ impl InputWorker { will_probe, series_probe_settings, query_opt, + Some(&tmdb_and_date_present), ) .await { Ok(Some(props)) => { collector.add_series(id.clone(), props); - Ok(()) + Ok(tmdb_and_date_present.load(Ordering::Relaxed)) } - Ok(None) => Ok(()), + Ok(None) => Ok(tmdb_and_date_present.load(Ordering::Relaxed)), Err(e) => Err(e), } } @@ -2839,9 +2889,9 @@ impl InputWorker { { Ok(Some(props)) => { collector.add_live(id.clone(), props); - Ok(()) + Ok(false) } - Ok(None) => Ok(()), + Ok(None) => Ok(false), Err(e) => Err(e), } } @@ -2866,7 +2916,7 @@ impl InputWorker { .await?; match outcome { - GenericProbeOutcome::Updated | GenericProbeOutcome::Noop => Ok(()), + GenericProbeOutcome::Updated | GenericProbeOutcome::Noop => Ok(false), GenericProbeOutcome::ProbeFailed => Err(shared::error::info_err!( "Probe stream task failed for key {:?} ({})", task_key, @@ -2884,6 +2934,37 @@ mod tests { use tempfile::tempdir; use tokio::sync::mpsc; + fn create_test_worker( + input_name: &str, + sender: mpsc::Sender, + receiver: mpsc::Receiver, + pending_tasks: Arc>, + pending_task_count: Arc, + ) -> InputWorker { + InputWorker { + input_name: Arc::from(input_name), + sender, + receiver, + pending_tasks, + pending_task_count, + app_state_weak: None, + update_pause_gate: Arc::new(RwLock::new(())), + cancel_token: CancellationToken::new(), + batch_buffer: BatchResultCollector::new(), + db_handles: HashMap::new(), + failed_clusters: HashSet::new(), + retry_states: HashMap::new(), + resolve_exhausted: HashMap::new(), + last_cycle_completed_at_ts: None, + metadata_retry_state_path: None, + metadata_retry_loaded: false, + metadata_retry_load_retry_at_ts: None, + last_retry_state_prune_at_ts: None, + scheduled_requeues: Arc::new(DashMap::new()), + recently_completed_no_change: HashMap::new(), + } + } + #[tokio::test] async fn queue_task_creates_single_worker_per_input_under_concurrency() { let cancel_token = CancellationToken::new(); @@ -3132,6 +3213,7 @@ mod tests { async fn finalize_processed_task_success_requeues_when_generation_changed() { let (tx, rx) = mpsc::channel::(8); let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); let key = TaskKey::Vod(7); pending_tasks.insert( @@ -3146,27 +3228,7 @@ mod tests { entry.generation.store(1, Ordering::Relaxed); } - let mut worker = InputWorker { - input_name: Arc::from("input_a"), - sender: tx, - receiver: rx, - pending_tasks: pending_tasks.clone(), - pending_task_count: Arc::new(AtomicUsize::new(1)), - app_state_weak: None, - update_pause_gate: Arc::new(RwLock::new(())), - cancel_token: CancellationToken::new(), - batch_buffer: BatchResultCollector::new(), - db_handles: HashMap::new(), - failed_clusters: HashSet::new(), - retry_states: HashMap::new(), - resolve_exhausted: HashMap::new(), - last_cycle_completed_at_ts: None, - metadata_retry_state_path: None, - metadata_retry_loaded: false, - metadata_retry_load_retry_at_ts: None, - last_retry_state_prune_at_ts: None, - scheduled_requeues: Arc::new(DashMap::new()), - }; + let mut worker = create_test_worker("input_a", tx, rx, pending_tasks.clone(), pending_task_count); let requeued = worker.finalize_processed_task_success(&key, 0, "input_a").await; assert!(requeued); @@ -3178,45 +3240,101 @@ mod tests { async fn finalize_processed_task_success_removes_when_unchanged() { let (tx, rx) = mpsc::channel::(8); let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); let key = TaskKey::Vod(9); - - pending_tasks.insert( - key.clone(), - PendingTask::new(UpdateTask::ResolveVod { - id: ProviderIdType::Id(9), - reason: ResolveReasonSet::from_variants(&[ResolveReason::Info]), - delay: 0, - }), - ); - - let mut worker = InputWorker { - input_name: Arc::from("input_b"), - sender: tx, - receiver: rx, - pending_tasks: pending_tasks.clone(), - pending_task_count: Arc::new(AtomicUsize::new(1)), - app_state_weak: None, - update_pause_gate: Arc::new(RwLock::new(())), - cancel_token: CancellationToken::new(), - batch_buffer: BatchResultCollector::new(), - db_handles: HashMap::new(), - failed_clusters: HashSet::new(), - retry_states: HashMap::new(), - resolve_exhausted: HashMap::new(), - last_cycle_completed_at_ts: None, - metadata_retry_state_path: None, - metadata_retry_loaded: false, - metadata_retry_load_retry_at_ts: None, - last_retry_state_prune_at_ts: None, - scheduled_requeues: Arc::new(DashMap::new()), + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(9), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Info]), + delay: 0, }; + pending_tasks.insert(key.clone(), PendingTask::new(task.clone())); + + let mut worker = create_test_worker("input_b", tx, rx, pending_tasks.clone(), pending_task_count); + let runtime_settings = MetadataUpdateRuntimeSettings::default(); + + worker.scheduled_requeues.insert(key.clone(), chrono::Utc::now().timestamp().saturating_add(30)); + worker.recently_completed_no_change.insert( + key.clone(), + (Instant::now(), ResolveReasonSet::from_variants(&[ResolveReason::Info])), + ); + assert!(worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(worker.recently_completed_no_change.contains_key(&key)); + assert!(!worker.scheduled_requeues.contains_key(&key)); + let requeued = worker.finalize_processed_task_success(&key, 0, "input_b").await; assert!(!requeued); assert!(!pending_tasks.contains_key(&key)); assert!(matches!(worker.receiver.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Empty))); } + #[test] + fn recent_no_change_skip_requires_exact_reason_match() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let key = TaskKey::Vod(10); + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(10), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Info]), + delay: 0, + }; + let mut worker = create_test_worker("input_c", tx, rx, pending_tasks, pending_task_count); + let runtime_settings = MetadataUpdateRuntimeSettings::default(); + + worker.recently_completed_no_change.insert( + key.clone(), + ( + Instant::now(), + ResolveReasonSet::from_variants(&[ResolveReason::Info, ResolveReason::Probe]), + ), + ); + assert!(!worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(!worker.recently_completed_no_change.contains_key(&key)); + + worker.recently_completed_no_change.insert( + key.clone(), + (Instant::now(), ResolveReasonSet::from_variants(&[ResolveReason::Info])), + ); + assert!(worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(worker.recently_completed_no_change.contains_key(&key)); + } + + #[tokio::test] + async fn recent_no_change_skip_ttl_expiry_allows_requeue() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let key = TaskKey::Vod(11); + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(11), + reason: ResolveReasonSet::from_variants(&[ResolveReason::Info]), + delay: 0, + }; + + pending_tasks.insert(key.clone(), PendingTask::new(task.clone())); + if let Some(entry) = pending_tasks.get(&key) { + entry.generation.store(1, Ordering::Relaxed); + } + + let mut worker = create_test_worker("input_d", tx, rx, pending_tasks.clone(), pending_task_count); + let mut runtime_settings = MetadataUpdateRuntimeSettings::default(); + runtime_settings.no_change_cache_ttl_secs = 1; + + let stale_instant = Instant::now().checked_sub(Duration::from_secs(2)).unwrap_or_else(Instant::now); + worker.recently_completed_no_change.insert( + key.clone(), + (stale_instant, ResolveReasonSet::from_variants(&[ResolveReason::Info])), + ); + assert!(!worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(!worker.recently_completed_no_change.contains_key(&key)); + + let requeued = worker.finalize_processed_task_success(&key, 0, "input_d").await; + assert!(requeued); + assert!(pending_tasks.contains_key(&key)); + assert_eq!(worker.receiver.try_recv().expect("requeued signal should be present"), key); + } + #[test] fn task_needs_provider_connection_skips_library_probe_stream() { let task = UpdateTask::ProbeStream { diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index 556d77595..667dfeb77 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -38,7 +38,7 @@ impl MetadataResolver { .api_key .as_ref() .map_or_else(|| TMDB_API_KEY.to_string(), ToString::to_string); - Some(TmdbClient::new(api_key, config.tmdb.rate_limit_ms, client, tmdb_storage)) + Some(TmdbClient::new(api_key, &config.tmdb, client, tmdb_storage)) } else { warn!("TMDB is enabled but metadata storage is unavailable; TMDB resolver is disabled."); None diff --git a/backend/src/library/tmdb.rs b/backend/src/library/tmdb.rs index b5b74ad82..b4c71edf4 100644 --- a/backend/src/library/tmdb.rs +++ b/backend/src/library/tmdb.rs @@ -227,7 +227,7 @@ impl TmdbSeriesInfoEpisodeDetails { season: self.season_number, episode: self.episode_number, aired: self.air_date.clone(), - plot: if self.overview.is_empty() {None} else { Some(self.overview.clone())}, + plot: if self.overview.is_empty() { None } else { Some(self.overview.clone()) }, runtime: self.runtime, rating: Some(self.vote_average), thumb: self.still_path.as_ref().and_then(|path| resolve_tmdb_image_url(path)), @@ -239,7 +239,7 @@ impl TmdbSeriesInfoEpisodeDetails { } #[derive(Debug, Deserialize)] -pub struct TmdbSeriesInfoSeasonDetails { +pub struct TmdbSeriesInfoSeasonDetails { // #[serde(rename = "_id")] // pub internal_id: String, // pub id: u32, @@ -453,7 +453,6 @@ pub struct TmdbSeason { } impl TmdbSeason { - // TODO maybe use Arc for episodes, networks, credits to avoid memory usage pub fn to_meta_data(&self) -> SeasonMetadata { SeasonMetadata { @@ -462,7 +461,7 @@ impl TmdbSeason { episode_count: self.episode_count, name: self.name.clone(), overview: self.overview.clone(), - poster_path: self.poster_path.clone(), + poster_path: self.poster_path.as_ref().and_then(|p| resolve_tmdb_image_url(p)), season_number: self.season_number, vote_average: self.vote_average, episodes: self.episodes.clone(), @@ -474,7 +473,7 @@ impl TmdbSeason { #[cfg(test)] mod tests { - use super::{resolve_tmdb_image_url, TMDB_IMAGE_BASE_URL}; + use super::{resolve_tmdb_image_url, TmdbMovieDetails, TMDB_IMAGE_BASE_URL}; #[test] fn test_resolve_tmdb_url_with_absolute_http() { @@ -511,4 +510,12 @@ mod tests { let input = " "; assert_eq!(resolve_tmdb_image_url(input), None); } + + // #[test] + // fn test_parse_movie_details() { + // curl "https://api.themoviedb.org/3/movie/84892?api_key=4219e299c89411838049ab0dab19ebd5&append_to_response=credits,videos,external_ids" + // let a: TmdbMovieDetails = serde_json::from_str(RESULT).map_err(|err| format!("Failed to parse TMDB movie details: {err}")).expect("Failed to parse TMDB movie details"); + // assert_eq!(a.id, 84892) + // } } + diff --git a/backend/src/library/tmdb_client.rs b/backend/src/library/tmdb_client.rs index a8663dfd0..d97920851 100644 --- a/backend/src/library/tmdb_client.rs +++ b/backend/src/library/tmdb_client.rs @@ -15,14 +15,13 @@ use std::{ }; use tokio::time::{sleep, timeout, Duration, Instant}; use url::Url; +use crate::model::TmdbConfig; // TODO make this configurable in Library tmdb config const TMDB_API_BASE_URL: &str = "https://api.themoviedb.org/3"; const MAX_RETRIES: u32 = 3; const REQUEST_TIMEOUT_SECS: u64 = 30; const MAX_FETCHED_CACHE_ENTRIES: usize = 10_000; -/// Minimum Jaro-Winkler score required to accept a TMDB search result. -const TMDB_MATCH_THRESHOLD: f64 = 0.9; /// Bounded set with insertion-order (FIFO) eviction. /// This is intentionally not LRU: `contains()` does not refresh recency. @@ -84,6 +83,7 @@ pub struct TmdbClient { api_key: String, client: reqwest::Client, rate_limit_ms: u64, + match_threshold: f64, storage: MetadataStorage, fetched_movie_metadata: tokio::sync::RwLock>, fetched_series_metadata: tokio::sync::RwLock>, @@ -92,11 +92,13 @@ pub struct TmdbClient { impl TmdbClient { // Creates a new TMDB client - pub fn new(api_key: String, rate_limit_ms: u64, client: reqwest::Client, storage: MetadataStorage) -> Self { + pub fn new(api_key: String, tmdb_config: &TmdbConfig, client: reqwest::Client, storage: MetadataStorage) -> Self { + let capped_match_threshold = tmdb_config.match_threshold.clamp(0, 100); Self { api_key, client, - rate_limit_ms, + rate_limit_ms: tmdb_config.rate_limit_ms, + match_threshold: f64::from(capped_match_threshold) / 100.0f64, storage, fetched_movie_metadata: tokio::sync::RwLock::new(BoundedSet::new(MAX_FETCHED_CACHE_ENTRIES)), fetched_series_metadata: tokio::sync::RwLock::new(BoundedSet::new(MAX_FETCHED_CACHE_ENTRIES)), @@ -268,11 +270,11 @@ impl TmdbClient { .map(|m| (m.id, m.title.as_str(), m.original_title.as_str())) .collect(); - if let Some((score, movie_id)) = Self::best_match_by_title(&query_lower, &candidates) { + if let Some((score, movie_id)) = Self::best_match_by_title(&query_lower, &candidates, self.match_threshold) { debug!("TMDB movie best match for '{title}': ID {movie_id} (score {score:.2})"); self.fetch_movie_details(movie_id).await } else { - debug!("TMDB movie search for '{title}': no result met threshold {TMDB_MATCH_THRESHOLD:.2}"); + debug!("TMDB movie search for '{title}': no result met threshold {:.2}", self.match_threshold); Ok(None) } } else { @@ -390,11 +392,11 @@ impl TmdbClient { .map(|s| (s.id, s.name.as_str(), s.original_name.as_str())) .collect(); - if let Some((score, series_id)) = Self::best_match_by_title(&query_lower, &candidates) { + if let Some((score, series_id)) = Self::best_match_by_title(&query_lower, &candidates, self.match_threshold) { debug!("TMDB series best match for '{title}': ID {series_id} (score {score:.2})"); self.fetch_series_details(series_id).await } else { - debug!("TMDB series search for '{title}': no result met threshold {TMDB_MATCH_THRESHOLD:.2}"); + debug!("TMDB series search for '{title}': no result met threshold {:.2}", self.match_threshold); Ok(None) } } else { @@ -406,8 +408,9 @@ impl TmdbClient { /// /// Compares `query` (already lowercased) against both the primary title and /// the original title of every candidate, takes the maximum score, and returns - /// `Some((score, id))` only when the best score is >= `TMDB_MATCH_THRESHOLD`. - fn best_match_by_title(query: &str, candidates: &[(u32, &str, &str)]) -> Option<(f64, u32)> { + /// `Some((score, id))` only when the best score is >= `match_threshold`. + fn best_match_by_title(query: &str, candidates: &[(u32, &str, &str)], match_threshold: f64) -> Option<(f64, u32)> { + candidates .iter() .map(|&(id, title, original_title)| { @@ -416,7 +419,7 @@ impl TmdbClient { (score, id) }) .max_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)) - .filter(|&(score, _)| score >= TMDB_MATCH_THRESHOLD) + .filter(|&(score, _)| score >= match_threshold) } // Fetches detailed TV series information @@ -589,9 +592,3 @@ impl TmdbClient { } } } - -#[cfg(test)] -mod tests { - const RESULT: &str = r#"{"adult":false,"backdrop_path":"/aM6E4DBP6588q3tEr9hz41ls80q.jpg","belongs_to_collection":null,"budget":13000000,"genres":[{"id":18,"name":"Drama"}],"homepage":"https://www.lionsgate.com/movies/the-perks-of-being-a-wallflower","id":84892,"imdb_id":"tt1659337","origin_country":["US"],"original_language":"en","original_title":"The Perks of Being a Wallflower","overview":"Pittsburgh, Pennsylvania, 1991. High school freshman Charlie is a wallflower, always watching life from the sidelines, until two senior students, Sam and her stepbrother Patrick, become his mentors, helping him discover the joys of friendship, music and love.","popularity":5.8295,"poster_path":"/aKCvdFFF5n80P2VdS7d8YBwbCjh.jpg","production_companies":[{"id":2130,"logo_path":"/g0lqeY2FvhzXcOI6z8RVXbORRUY.png","name":"Mr. Mudd","origin_country":"US"}],"production_countries":[{"iso_3166_1":"US","name":"United States of America"}],"release_date":"2012-09-20","revenue":33384127,"runtime":103,"spoken_languages":[{"english_name":"English","iso_639_1":"en","name":"English"}],"status":"Released","tagline":"We are infinite.","title":"The Perks of Being a Wallflower","video":false,"vote_average":7.803,"vote_count":11045,"credits":{"cast":[{"adult":false,"gender":2,"id":33235,"known_for_department":"Acting","name":"Logan Lerman","original_name":"Logan Lerman","popularity":2.7882,"profile_path":"/wEte1WtpzwNzhA9adib1rJGbTvb.jpg","cast_id":6,"character":"Charlie","credit_id":"52fe49169251416c910a23af","order":0},{"adult":false,"gender":1,"id":10990,"known_for_department":"Acting","name":"Emma Watson","original_name":"Emma Watson","popularity":8.9879,"profile_path":"/A14lLCZYDhfYdBa0fFRpwMDiwRN.jpg","cast_id":12,"character":"Sam","credit_id":"53173900c3a368136e0029f1","order":1},{"adult":false,"gender":3,"id":132157,"known_for_department":"Acting","name":"Ezra Miller","original_name":"Ezra Miller","popularity":3.2172,"profile_path":"/hLtxNK8eeWZkFSeaAASFWm15Qv0.jpg","cast_id":9,"character":"Patrick","credit_id":"52fe49169251416c910a23bb","order":2},{"adult":false,"gender":1,"id":52404,"known_for_department":"Acting","name":"Mae Whitman","original_name":"Mae Whitman","popularity":2.2888,"profile_path":"/x0DdzjoYN8K2PwjrnH3ogPYv2zo.jpg","cast_id":8,"character":"Mary Elizabeth","credit_id":"52fe49169251416c910a23b7","order":3},{"adult":false,"gender":1,"id":61114,"known_for_department":"Acting","name":"Kate Walsh","original_name":"Kate Walsh","popularity":2.0713,"profile_path":"/jRg4pmjvI2063YnQ6PJidclOc4L.jpg","cast_id":14,"character":"Mother","credit_id":"531739709251412ccd001076","order":4},{"adult":false,"gender":2,"id":32597,"known_for_department":"Acting","name":"Dylan McDermott","original_name":"Dylan McDermott","popularity":1.853,"profile_path":"/3i69RNLuL7KOD9pCmHQYruhcOdn.jpg","cast_id":32,"character":"Father","credit_id":"55992a9a9251413d96002e93","order":5},{"adult":false,"gender":1,"id":15091,"known_for_department":"Acting","name":"Melanie Lynskey","original_name":"Melanie Lynskey","popularity":2.084,"profile_path":"/kzrWI1sTgnA0H7TCIKzDOUtOW4n.jpg","cast_id":10,"character":"Aunt Helen","credit_id":"52fe49169251416c910a23bf","order":6},{"adult":false,"gender":1,"id":19961,"known_for_department":"Acting","name":"Nina Dobrev","original_name":"Nina Dobrev","popularity":3.3999,"profile_path":"/67A1s3I8k831MJ7VRLX59hBNdNt.jpg","cast_id":4,"character":"Candace","credit_id":"52fe49169251416c910a23a7","order":7},{"adult":false,"gender":2,"id":27104,"known_for_department":"Acting","name":"Johnny Simmons","original_name":"Johnny Simmons","popularity":1.0991,"profile_path":"/51LZiAAI3ZW4vabNOQnd3weYgUm.jpg","cast_id":13,"character":"Brad","credit_id":"5317393b9251415861001b5d","order":8},{"adult":false,"gender":1,"id":3234,"known_for_department":"Acting","name":"Joan Cusack","original_name":"Joan Cusack","popularity":2.3403,"profile_path":"/69cfjfZFjVxfu2QbngnXOkipcyn.jpg","cast_id":15,"character":"Dr. Burton","credit_id":"5317399f92514158a0001c3f","order":9},{"adult":false,"gender":2,"id":22226,"known_for_department":"Acting","name":"Paul Rudd","original_name":"Paul Rudd","popularity":2.8312,"profile_path":"/6jtwNOLKy0LdsRAKwZqgYMAfd5n.jpg","cast_id":7,"character":"Mr. Anderson","credit_id":"52fe49169251416c910a23b3","order":10},{"adult":false,"gender":2,"id":85139,"known_for_department":"Acting","name":"Nicholas Braun","original_name":"Nicholas Braun","popularity":1.5594,"profile_path":"/b2I6bZptuld3pjlVkYIy4DtMKGg.jpg","cast_id":18,"character":"Ponytail Derek","credit_id":"54984f1c9251417a810060e6","order":11},{"adult":false,"gender":2,"id":48463,"known_for_department":"Acting","name":"Reece Thompson","original_name":"Reece Thompson","popularity":0.5843,"profile_path":"/xA212pV7L7FcF6yxO24RGEOiHWb.jpg","cast_id":35,"character":"Craig","credit_id":"5905937492514169d0019096","order":12},{"adult":false,"gender":0,"id":1107313,"known_for_department":"Acting","name":"Patrick de Ledebur","original_name":"Patrick de Ledebur","popularity":0.5647,"profile_path":null,"cast_id":44,"character":"Senior Bully","credit_id":"5e718db88de0ae0013553923","order":13},{"adult":false,"gender":0,"id":227229,"known_for_department":"Acting","name":"Brian Balzerini","original_name":"Brian Balzerini","popularity":0.2854,"profile_path":null,"cast_id":45,"character":"Linebacker","credit_id":"5e718dee357c000016473a3a","order":14},{"adult":false,"gender":0,"id":2569643,"known_for_department":"Acting","name":"Tom Kruszewski","original_name":"Tom Kruszewski","popularity":0.1167,"profile_path":null,"cast_id":46,"character":"Nose Tackle","credit_id":"5e718e01b1f68d0014dbdf1b","order":15},{"adult":false,"gender":1,"id":936970,"known_for_department":"Acting","name":"Julia Garner","original_name":"Julia Garner","popularity":2.0298,"profile_path":"/ud1RXbvW70J89iqeic7no8olxvb.jpg","cast_id":17,"character":"Susan","credit_id":"5381f182c3a368737d00224e","order":16},{"adult":false,"gender":2,"id":11161,"known_for_department":"Costume & Make-Up","name":"Tom Savini","original_name":"Tom Savini","popularity":0.8798,"profile_path":"/zBYnzxzlAIEoanEU00bGYJmRS6k.jpg","cast_id":31,"character":"Mr. Callahan","credit_id":"559929fc9251413d96002e84","order":17},{"adult":false,"gender":0,"id":2569647,"known_for_department":"Acting","name":"Emily Marie Callaway","original_name":"Emily Marie Callaway","popularity":0.215,"profile_path":null,"cast_id":47,"character":"Mean Freshman Girl","credit_id":"5e718e808de0ae001a553d4a","order":18},{"adult":false,"gender":1,"id":1456334,"known_for_department":"Acting","name":"Chelsea Zhang","original_name":"Chelsea Zhang","popularity":0.2055,"profile_path":"/1bI5RT3LST8qDxkgBWH2M8AdZVW.jpg","cast_id":48,"character":"Shakespeare Girl","credit_id":"5e718ea9b1f68d0012dbb177","order":19},{"adult":false,"gender":0,"id":2569648,"known_for_department":"Acting","name":"Jesse Scheirer","original_name":"Jesse Scheirer","popularity":0.1367,"profile_path":null,"cast_id":49,"character":"Freshman Boy","credit_id":"5e718ec12f3b170014486f29","order":20},{"adult":false,"gender":0,"id":2569649,"known_for_department":"Acting","name":"Justine Nicole Schaefer","original_name":"Justine Nicole Schaefer","popularity":0.042,"profile_path":null,"cast_id":50,"character":"Twin Girl #1","credit_id":"5e718eeacabfe4001518b2a5","order":21},{"adult":false,"gender":0,"id":2569650,"known_for_department":"Acting","name":"Julie Marie Schaefer","original_name":"Julie Marie Schaefer","popularity":0.2808,"profile_path":null,"cast_id":51,"character":"Twin Girl #2","credit_id":"5e718efc357c00001347a50d","order":22},{"adult":false,"gender":0,"id":2569651,"known_for_department":"Acting","name":"Leo Miles Farmerie","original_name":"Leo Miles Farmerie","popularity":0.1415,"profile_path":null,"cast_id":52,"character":"7-Year-Old Charlie","credit_id":"5e718f54b1f68d0019dc2700","order":23},{"adult":false,"gender":0,"id":2569652,"known_for_department":"Acting","name":"Isabel Muschweck","original_name":"Isabel Muschweck","popularity":0.0984,"profile_path":null,"cast_id":53,"character":"9-Year-Old Candace","credit_id":"5e718f658de0ae0017556b09","order":24},{"adult":false,"gender":2,"id":1517251,"known_for_department":"Acting","name":"Adam Hagenbuch","original_name":"Adam Hagenbuch","popularity":1.2086,"profile_path":"/42CrwdFz2VuFHxzxUTIaHGnjltN.jpg","cast_id":34,"character":"Bob","credit_id":"5905936692514169c80195dc","order":25},{"adult":false,"gender":1,"id":1053419,"known_for_department":"Acting","name":"Erin Wilhelmi","original_name":"Erin Wilhelmi","popularity":1.1874,"profile_path":"/nj3nWlDsWbVIJiFeL8TB0VwYkDE.jpg","cast_id":16,"character":"Alice","credit_id":"531739c992514158a0001c42","order":26},{"adult":false,"gender":0,"id":2563015,"known_for_department":"Directing","name":"Jordan Paley","original_name":"Jordan Paley","popularity":0.4119,"profile_path":null,"cast_id":54,"character":"Rocky MC","credit_id":"5e71905bcabfe4001518bbad","order":27},{"adult":false,"gender":2,"id":987572,"known_for_department":"Acting","name":"Zane Holtz","original_name":"Zane Holtz","popularity":0.7137,"profile_path":"/z5sXU2zOR59A3kfJTgwPObgrPhf.jpg","cast_id":55,"character":"Chris","credit_id":"5e71906f357c00001947a831","order":28},{"adult":false,"gender":0,"id":2569659,"known_for_department":"Acting","name":"Timothy Breslin","original_name":"Timothy Breslin","popularity":0.0652,"profile_path":null,"cast_id":56,"character":"Policeman","credit_id":"5e719086357c00001947a865","order":29},{"adult":false,"gender":2,"id":1432326,"known_for_department":"Acting","name":"Mark McClain Wilson","original_name":"Mark McClain Wilson","popularity":0.1199,"profile_path":"/dvReWjPgr8Y8B6iZv9NiBDYuAJl.jpg","cast_id":57,"character":"Emergency Room Policeman","credit_id":"5e71909cf9aa470013cfdacf","order":30},{"adult":false,"gender":2,"id":1129400,"known_for_department":"Acting","name":"Atticus Cain","original_name":"Atticus Cain","popularity":0.7613,"profile_path":"/zTMpCXOND4zxca2pOEncm2UfmNO.jpg","cast_id":58,"character":"Emergency Room Doctor","credit_id":"5e7190b2b1f68d0012dbc391","order":31},{"adult":false,"gender":1,"id":120356,"known_for_department":"Acting","name":"Stacy Chbosky","original_name":"Stacy Chbosky","popularity":0.4934,"profile_path":"/m4A1RZc8GelxBufn9Ehqv7VcKZh.jpg","cast_id":59,"character":"Young Mom","credit_id":"5e7190c18de0ae0017556cfc","order":32},{"adult":false,"gender":2,"id":207069,"known_for_department":"Acting","name":"Dihlon McManne","original_name":"Dihlon McManne","popularity":0.6182,"profile_path":null,"cast_id":60,"character":"Priest","credit_id":"5e7190d12f3b17001148ab92","order":33},{"adult":false,"gender":0,"id":1537118,"known_for_department":"Acting","name":"Laurie Klatscher","original_name":"Laurie Klatscher","popularity":0.5801,"profile_path":null,"cast_id":61,"character":"School Principal","credit_id":"5e7190ec8de0ae0013553c93","order":34},{"adult":false,"gender":2,"id":1231874,"known_for_department":"Acting","name":"Landon Pigg","original_name":"Landon Pigg","popularity":0.4364,"profile_path":"/705aTqQe4FMswVAcIPOZZQ6JNnT.jpg","cast_id":62,"character":"Peter","credit_id":"5e7190fe357c00001347a7d5","order":35},{"adult":false,"gender":1,"id":1555395,"known_for_department":"Acting","name":"Jennifer Enskat","original_name":"Jennifer Enskat","popularity":0.7623,"profile_path":"/aibiuaBXq9W65QgEUz0ZcJHjfKW.jpg","cast_id":63,"character":"Sam's Mom","credit_id":"5e719113f9aa470015cf85d8","order":36},{"adult":false,"gender":2,"id":1605510,"known_for_department":"Acting","name":"William L. Thomas","original_name":"William L. Thomas","popularity":0.1091,"profile_path":"/rNhe83bSwTvHtYHTMufUkmunBDK.jpg","cast_id":64,"character":"Patrick's Dad","credit_id":"5e7191292f3b17001148ac67","order":37},{"adult":false,"gender":1,"id":1205752,"known_for_department":"Acting","name":"Morgan Wolk","original_name":"Morgan Wolk","popularity":1.6609,"profile_path":"/pdQiuPTihAm6NERSDcqaxjQ1ULs.jpg","cast_id":65,"character":"Candace's Friend","credit_id":"5e71914bcabfe4001518bcfb","order":38},{"adult":false,"gender":2,"id":1543874,"known_for_department":"Acting","name":"Joe Fishel","original_name":"Joe Fishel","popularity":0.9006,"profile_path":"/Aapim9EAy3vY8eeS4n4tnF4iHsY.jpg","cast_id":108,"character":"Father of Twin Girls (uncredited)","credit_id":"65d4cf0eb9a0bd0186684338","order":39}],"crew":[{"adult":false,"gender":2,"id":19311,"known_for_department":"Writing","name":"Stephen Chbosky","original_name":"Stephen Chbosky","popularity":0.6439,"profile_path":"/9PdTBjn8dJqfn3ygKvsCdl9G06J.jpg","credit_id":"52fe49169251416c910a2397","department":"Directing","job":"Director"},{"adult":false,"gender":2,"id":19311,"known_for_department":"Writing","name":"Stephen Chbosky","original_name":"Stephen Chbosky","popularity":0.6439,"profile_path":"/9PdTBjn8dJqfn3ygKvsCdl9G06J.jpg","credit_id":"52fe49169251416c910a23a3","department":"Writing","job":"Screenplay"},{"adult":false,"gender":1,"id":1329415,"known_for_department":"Crew","name":"Samantha MacIvor","original_name":"Samantha MacIvor","popularity":0.565,"profile_path":"/ztH9qxR7mknj8Rs2MCNhayA7n5l.jpg","credit_id":"64ee96e9caa50800c885cda5","department":"Crew","job":"Stunts"},{"adult":false,"gender":2,"id":6949,"known_for_department":"Acting","name":"John Malkovich","original_name":"John Malkovich","popularity":2.8206,"profile_path":"/7GoOdGNc4ra1L0F5nJTkmIB37iu.jpg","credit_id":"531738bcc3a36813a60028a2","department":"Production","job":"Producer"},{"adult":false,"gender":2,"id":19016,"known_for_department":"Sound","name":"Michael Brook","original_name":"Michael Brook","popularity":0.217,"profile_path":"/PbHAuMBPkxtgyEmV6d69BTj2fZ.jpg","credit_id":"553108b7c3a368412100215a","department":"Sound","job":"Original Music Composer"},{"adult":false,"gender":1,"id":15350,"known_for_department":"Editing","name":"Mary Jo Markey","original_name":"Mary Jo Markey","popularity":0.5209,"profile_path":"/lfGOI3S9JModCwIgqsYBB3kP2NV.jpg","credit_id":"55310807c3a3680a94001eb8","department":"Editing","job":"Editor"},{"adult":false,"gender":2,"id":8846,"known_for_department":"Camera","name":"Andrew Dunn","original_name":"Andrew Dunn","popularity":0.7751,"profile_path":"/j7ShQfHmJtz0Pw2tMpYEvSKkqGL.jpg","credit_id":"5531084ec3a3684112002224","department":"Camera","job":"Director of Photography"},{"adult":false,"gender":1,"id":52445,"known_for_department":"Production","name":"Lianne Halfon","original_name":"Lianne Halfon","popularity":0.2749,"profile_path":null,"credit_id":"553109edc3a3680f420017f8","department":"Production","job":"Producer"},{"adult":false,"gender":2,"id":19311,"known_for_department":"Writing","name":"Stephen Chbosky","original_name":"Stephen Chbosky","popularity":0.6439,"profile_path":"/9PdTBjn8dJqfn3ygKvsCdl9G06J.jpg","credit_id":"55310a53c3a36841120022b7","department":"Production","job":"Executive Producer"},{"adult":false,"gender":1,"id":1193617,"known_for_department":"Art","name":"Inbal Weinberg","original_name":"Inbal Weinberg","popularity":1.6401,"profile_path":"/6vXjNLJJLjCnkd1IEMjuH92zVFT.jpg","credit_id":"553109d1c3a3680a94001f3b","department":"Art","job":"Production Design"},{"adult":false,"gender":2,"id":52897,"known_for_department":"Production","name":"Russell Smith","original_name":"Russell Smith","popularity":0.1128,"profile_path":null,"credit_id":"55310a1bc3a3680f42001807","department":"Production","job":"Producer"},{"adult":false,"gender":2,"id":82132,"known_for_department":"Production","name":"James Powers","original_name":"James Powers","popularity":0.4909,"profile_path":null,"credit_id":"55310f8b9251410675000001","department":"Production","job":"Executive Producer"},{"adult":false,"gender":1,"id":1015922,"known_for_department":"Directing","name":"Diane Hassinger Newman","original_name":"Diane Hassinger Newman","popularity":0.5791,"profile_path":null,"credit_id":"5532163cc3a36848ca000ea9","department":"Directing","job":"Script Supervisor"},{"adult":false,"gender":1,"id":39123,"known_for_department":"Production","name":"Venus Kanani","original_name":"Venus Kanani","popularity":0.6783,"profile_path":"/lXCI0CoU6FvJqKcHN32Yq6ilbcP.jpg","credit_id":"553215e4c3a368222a00202f","department":"Production","job":"Casting"},{"adult":false,"gender":1,"id":5914,"known_for_department":"Production","name":"Mary Vernieu","original_name":"Mary Vernieu","popularity":0.9105,"profile_path":"/z37Cmn0MJdWCSC8ydkyoseiYUYk.jpg","credit_id":"5532158c92514163100017f2","department":"Production","job":"Casting"},{"adult":false,"gender":1,"id":1521494,"known_for_department":"Costume & Make-Up","name":"Diane Collins","original_name":"Diane Collins","popularity":0.4266,"profile_path":null,"credit_id":"637b10c5336e010082e206c1","department":"Costume & Make-Up","job":"Costume Supervisor"},{"adult":false,"gender":0,"id":2773778,"known_for_department":"Art","name":"Aaron Streiner","original_name":"Aaron Streiner","popularity":0.1377,"profile_path":null,"credit_id":"637b11372cde980075ad58af","department":"Art","job":"Set Dresser"},{"adult":false,"gender":2,"id":1432038,"known_for_department":"Visual Effects","name":"Phillip Hoffman","original_name":"Phillip Hoffman","popularity":0.4458,"profile_path":null,"credit_id":"637b10e82cde9800cc15e789","department":"Visual Effects","job":"Visual Effects Producer"},{"adult":false,"gender":0,"id":1395032,"known_for_department":"Costume & Make-Up","name":"Amanda Jenkins","original_name":"Amanda Jenkins","popularity":0.2001,"profile_path":null,"credit_id":"637b10b45b2f4700d58382a6","department":"Costume & Make-Up","job":"Set Costumer"},{"adult":false,"gender":2,"id":1316448,"known_for_department":"Art","name":"Thomas F. Kelly","original_name":"Thomas F. Kelly","popularity":0.2431,"profile_path":null,"credit_id":"637b1111156cc7009435f2a0","department":"Art","job":"Set Dresser"},{"adult":false,"gender":1,"id":2053858,"known_for_department":"Costume & Make-Up","name":"Melanie Marie Evans","original_name":"Melanie Marie Evans","popularity":0.3429,"profile_path":null,"credit_id":"637b10cf156cc7009435f27c","department":"Costume & Make-Up","job":"Set Costumer"},{"adult":false,"gender":0,"id":2773759,"known_for_department":"Art","name":"Eugene Doyle","original_name":"Eugene Doyle","popularity":0.2853,"profile_path":null,"credit_id":"637b11065b2f47009ba9f7a0","department":"Art","job":"Set Dresser"},{"adult":false,"gender":1,"id":1145972,"known_for_department":"Art","name":"Merissa Lombardo","original_name":"Merissa Lombardo","popularity":0.4407,"profile_path":null,"credit_id":"5e7192d52f3b170017488727","department":"Art","job":"Set Decoration"},{"adult":false,"gender":2,"id":1371064,"known_for_department":"Sound","name":"Gregg Barbanell","original_name":"Gregg Barbanell","popularity":0.4921,"profile_path":null,"credit_id":"5e7194b9f9aa470019cfb740","department":"Sound","job":"Foley Artist"},{"adult":false,"gender":0,"id":1557612,"known_for_department":"Sound","name":"Jeffree Bloomer","original_name":"Jeffree Bloomer","popularity":0.2458,"profile_path":null,"credit_id":"5e7192318de0ae001a554255","department":"Sound","job":"Sound Mixer"},{"adult":false,"gender":0,"id":1636644,"known_for_department":"Art","name":"Christina Myal","original_name":"Christina Myal","popularity":0.1342,"profile_path":null,"credit_id":"5e7192eccabfe4001318eafe","department":"Art","job":"Graphic Designer"},{"adult":false,"gender":0,"id":2034507,"known_for_department":"Costume & Make-Up","name":"Patty Bell","original_name":"Patty Bell","popularity":0.1535,"profile_path":null,"credit_id":"5e71931ecabfe4001118e767","department":"Costume & Make-Up","job":"Key Makeup Artist"},{"adult":false,"gender":1,"id":1535770,"known_for_department":"Production","name":"Natalie Angel","original_name":"Natalie Angel","popularity":0.3398,"profile_path":null,"credit_id":"5e719379b1f68d0019dc2c66","department":"Production","job":"Production Coordinator"},{"adult":false,"gender":0,"id":1544667,"known_for_department":"Costume & Make-Up","name":"Nancy Keslar","original_name":"Nancy Keslar","popularity":0.2894,"profile_path":null,"credit_id":"5e719348cabfe4001518c232","department":"Costume & Make-Up","job":"Key Hair Stylist"},{"adult":false,"gender":2,"id":82132,"known_for_department":"Production","name":"James Powers","original_name":"James Powers","popularity":0.4909,"profile_path":null,"credit_id":"5e718d2ef9aa470019cfa511","department":"Production","job":"Unit Production Manager"},{"adult":false,"gender":0,"id":2441835,"known_for_department":"Directing","name":"Susan Ransom-Coyle","original_name":"Susan Ransom-Coyle","popularity":0.3107,"profile_path":null,"credit_id":"5e718d668de0ae001a553b8b","department":"Directing","job":"Second Assistant Director"},{"adult":false,"gender":2,"id":162522,"known_for_department":"Crew","name":"Blaise Corrigan","original_name":"Blaise Corrigan","popularity":0.4892,"profile_path":"/ycsMR3gck2D8jJT6QEtzyrjUmC6.jpg","credit_id":"5e7191902f3b17001948f8df","department":"Crew","job":"Stunt Coordinator"},{"adult":false,"gender":2,"id":1319968,"known_for_department":"Crew","name":"Eric Bergman","original_name":"Eric Bergman","popularity":1.0646,"profile_path":null,"credit_id":"5e7193cacabfe4001318ec99","department":"Crew","job":"Post Production Supervisor"},{"adult":false,"gender":0,"id":1191813,"known_for_department":"Sound","name":"Noel Vought","original_name":"Noel Vought","popularity":0.5387,"profile_path":null,"credit_id":"5e71947ef9aa470015cf8c87","department":"Sound","job":"Foley Artist"},{"adult":false,"gender":2,"id":1406389,"known_for_department":"Sound","name":"Bruce Tanis","original_name":"Bruce Tanis","popularity":1.1468,"profile_path":null,"credit_id":"5e71946a8de0ae001a5548c6","department":"Sound","job":"Sound Editor"},{"adult":false,"gender":2,"id":1608764,"known_for_department":"Lighting","name":"Patrick Murray","original_name":"Patrick Murray","popularity":0.5066,"profile_path":null,"credit_id":"5e719254b1f68d0019dc2a99","department":"Lighting","job":"Gaffer"},{"adult":false,"gender":1,"id":1538148,"known_for_department":"Sound","name":"Alexandra Patsavas","original_name":"Alexandra Patsavas","popularity":0.5455,"profile_path":"/R9rHK6zVup8IXTO6gNgQ1JZ3nR.jpg","credit_id":"5e718c8e357c0000164738a1","department":"Sound","job":"Music Supervisor"},{"adult":false,"gender":2,"id":1391389,"known_for_department":"Camera","name":"John Bramley","original_name":"John Bramley","popularity":0.6375,"profile_path":null,"credit_id":"5e7191f9f9aa470015cf87c7","department":"Camera","job":"Still Photographer"},{"adult":false,"gender":0,"id":1545922,"known_for_department":"Sound","name":"Trevor Metz","original_name":"Trevor Metz","popularity":0.4125,"profile_path":null,"credit_id":"5e7194588de0ae001a55489f","department":"Sound","job":"Sound Editor"},{"adult":false,"gender":0,"id":2569663,"known_for_department":"Production","name":"Chris Gary","original_name":"Chris Gary","popularity":0.7371,"profile_path":"/pLhe9DdNytnega9dqLEKIcNkw1t.jpg","credit_id":"5e7191c8cabfe4001518bf30","department":"Production","job":"Associate Producer"},{"adult":false,"gender":2,"id":1564580,"known_for_department":"Sound","name":"Anthony Cargioli","original_name":"Anthony Cargioli","popularity":0.2092,"profile_path":null,"credit_id":"5e7192408de0ae001a554267","department":"Sound","job":"Boom Operator"},{"adult":false,"gender":0,"id":2569667,"known_for_department":"Sound","name":"Richard Dawn","original_name":"Richard Dawn","popularity":0.1517,"profile_path":null,"credit_id":"5e719446f9aa470017cfe307","department":"Sound","job":"Sound Editor"},{"adult":false,"gender":0,"id":1897204,"known_for_department":"Art","name":"Pete Dancy","original_name":"Pete Dancy","popularity":0.3924,"profile_path":null,"credit_id":"5e71928c2f3b17001748864e","department":"Art","job":"Property Master"},{"adult":false,"gender":2,"id":1533589,"known_for_department":"Visual Effects","name":"Russell Tyrrell","original_name":"Russell Tyrrell","popularity":0.0983,"profile_path":null,"credit_id":"5e7193a4f9aa470019cfb5cd","department":"Crew","job":"Special Effects Coordinator"},{"adult":false,"gender":2,"id":1367667,"known_for_department":"Sound","name":"Perry Robertson","original_name":"Perry Robertson","popularity":0.6849,"profile_path":null,"credit_id":"5e7193e7357c00001947ad06","department":"Sound","job":"Supervising Sound Editor"},{"adult":false,"gender":2,"id":1423757,"known_for_department":"Sound","name":"Scott Sanders","original_name":"Scott Sanders","popularity":0.4203,"profile_path":null,"credit_id":"5e7193fd8de0ae001a55480d","department":"Sound","job":"Supervising Sound Editor"},{"adult":false,"gender":2,"id":4190,"known_for_department":"Costume & Make-Up","name":"David C. Robinson","original_name":"David C. Robinson","popularity":0.4954,"profile_path":null,"credit_id":"5e718ca98de0ae001755663b","department":"Costume & Make-Up","job":"Costume Designer"},{"adult":false,"gender":0,"id":1208355,"known_for_department":"Art","name":"Gregory A. Weimerskirch","original_name":"Gregory A. Weimerskirch","popularity":0.4659,"profile_path":null,"credit_id":"5e7192b7f9aa470015cf8927","department":"Art","job":"Art Direction"},{"adult":false,"gender":1,"id":1328146,"known_for_department":"Costume & Make-Up","name":"Evelyne Noraz","original_name":"Evelyne Noraz","popularity":0.4285,"profile_path":"/vWlfd0cO6zotDVKxFxSJQTFb8yP.jpg","credit_id":"5e71930dcabfe4001118e747","department":"Costume & Make-Up","job":"Makeup Department Head"},{"adult":false,"gender":0,"id":1852975,"known_for_department":"Production","name":"Gillian Brown","original_name":"Gillian Brown","popularity":0.3992,"profile_path":null,"credit_id":"5e7191b1cabfe4001518befb","department":"Production","job":"Co-Producer"},{"adult":false,"gender":2,"id":40119,"known_for_department":"Camera","name":"Keith Seymour","original_name":"Keith Seymour","popularity":0.1726,"profile_path":null,"credit_id":"5e71926f357c00001347aacb","department":"Camera","job":"Key Grip"},{"adult":false,"gender":1,"id":1406893,"known_for_department":"Costume & Make-Up","name":"Suzy Mazzarese-Allison","original_name":"Suzy Mazzarese-Allison","popularity":0.2112,"profile_path":null,"credit_id":"5e719338cabfe4001318ebbc","department":"Costume & Make-Up","job":"Hair Department Head"},{"adult":false,"gender":2,"id":1010751,"known_for_department":"Sound","name":"Joe Barnett","original_name":"Joe Barnett","popularity":0.6252,"profile_path":"/sA1hottTzB3uCz3jtYTURZkE0gM.jpg","credit_id":"5e719410357c000011475a2b","department":"Sound","job":"Sound Re-Recording Mixer"},{"adult":false,"gender":0,"id":1825167,"known_for_department":"Directing","name":"Chip Signore","original_name":"Chip Signore","popularity":0.6752,"profile_path":null,"credit_id":"5e718d49f9aa470019cfa54c","department":"Directing","job":"First Assistant Director"},{"adult":false,"gender":2,"id":1931337,"known_for_department":"Production","name":"Shawn Boyachek","original_name":"Shawn Boyachek","popularity":0.1506,"profile_path":null,"credit_id":"5e71929fb1f68d0012dbcadf","department":"Production","job":"Location Manager"},{"adult":false,"gender":0,"id":1432039,"known_for_department":"Visual Effects","name":"Adam Avitabile","original_name":"Adam Avitabile","popularity":0.2606,"profile_path":null,"credit_id":"5e7194ecf9aa470015cf8db6","department":"Visual Effects","job":"Visual Effects Supervisor"},{"adult":false,"gender":0,"id":1417005,"known_for_department":"Production","name":"Janice F. Sperling","original_name":"Janice F. Sperling","popularity":0.1941,"profile_path":null,"credit_id":"5e71935ecabfe4001518c260","department":"Production","job":"Production Supervisor"},{"adult":false,"gender":0,"id":2569665,"known_for_department":"Production","name":"Ava Dellaira","original_name":"Ava Dellaira","popularity":0.103,"profile_path":null,"credit_id":"5e7191e1cabfe4001318e916","department":"Production","job":"Associate Producer"},{"adult":false,"gender":0,"id":1402111,"known_for_department":"Sound","name":"Marshall Garlington","original_name":"Marshall Garlington","popularity":0.2226,"profile_path":null,"credit_id":"5e7194258de0ae0013553f92","department":"Sound","job":"Sound Re-Recording Mixer"},{"adult":false,"gender":2,"id":1114134,"known_for_department":"Acting","name":"Andy Partridge","original_name":"Andy Partridge","popularity":0.4647,"profile_path":null,"credit_id":"68d0685cca7c17934aac831a","department":"Sound","job":"Songs"},{"adult":false,"gender":2,"id":19311,"known_for_department":"Writing","name":"Stephen Chbosky","original_name":"Stephen Chbosky","popularity":0.6439,"profile_path":"/9PdTBjn8dJqfn3ygKvsCdl9G06J.jpg","credit_id":"698736166c26679a2ff80912","department":"Writing","job":"Book"}]},"videos":{"results":[{"iso_639_1":"en","iso_3166_1":"US","name":"'Charlie Takes One Last Ride' Scene | The Perks of Being a Wallflower","key":"jyneTS1B854","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-11T16:00:48.000Z","id":"66e1cc6704f4dd348c45624c"},{"iso_639_1":"en","iso_3166_1":"US","name":"'We Accept the Love We Think We Deserve' Scene | The Perks of Being a Wallflower","key":"AgUDpwAhwWg","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-10T16:00:10.000Z","id":"66e11c79c3ff8f970708144c"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Patrick & Charlie's Deep Conversation' Scene | The Perks of Being a Wallflower","key":"dbgf-5kPhEY","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-09T16:00:30.000Z","id":"66e11c714f3a71968054e156"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Charlie Kisses the Prettiest Girl in the Room' Scene | The Perks of Being a Wallflower","key":"GD6uSOrq7iY","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-08T16:00:10.000Z","id":"66e11c6a16773d46c95f00d7"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Sam & Charlie Perform at The Rocky Horror Picture Show' Scene | The Perks of Being A Wallflower","key":"vt7sUtJAK2c","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-07T16:01:01.000Z","id":"66e11c612c98375fa1051569"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Sam Helps Charlie Through a Bad Trip' Scene | The Perks of Being a Wallflower","key":"ebhHsFO4mts","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-06T16:00:42.000Z","id":"66e11c4d79ea57072f8fd1b3"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Charlie & Sam's First Kiss' Scene | The Perks of Being a Wallflower","key":"sehE3hKxwoM","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-05T16:00:49.000Z","id":"66e11c4300000000004c9022"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Driving Through the Tunnel' Scene | The Perks of Being a Wallflower","key":"avqZ2UMbc7Q","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-04T16:00:03.000Z","id":"66e11c354d6a14993435c3d2"},{"iso_639_1":"en","iso_3166_1":"US","name":"'The Homecoming Dance' Scene | The Perks of Being a Wallflower","key":"Y307eLrOcec","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-03T16:00:24.000Z","id":"66e11c2e000000000022a8f7"},{"iso_639_1":"en","iso_3166_1":"US","name":"'Charlie Meets Sam at the Football Game' Scene | The Perks of Being a Wallflower","key":"GyI_LYPoZAU","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-02T16:00:21.000Z","id":"66e11c2830b2e5c7af8fcfcf"},{"iso_639_1":"en","iso_3166_1":"US","name":"The First 10 Minutes of The Perks of Being a Wallflower (2012)","key":"qypWtUeaD40","site":"YouTube","size":1080,"type":"Clip","official":true,"published_at":"2024-09-01T16:00:51.000Z","id":"66e11c2179ea57072f8fd1a5"},{"iso_639_1":"en","iso_3166_1":"US","name":"DVD/BD Trailer","key":"x0nTfbg24Qs","site":"YouTube","size":720,"type":"Trailer","official":true,"published_at":"2013-01-04T19:56:58.000Z","id":"66e11b8b00000000004c9289"},{"iso_639_1":"en","iso_3166_1":"US","name":"The Perks of Love -- Perks of Being a Wallflower 2012","key":"sSlGwcqGO3g","site":"YouTube","size":480,"type":"Teaser","official":true,"published_at":"2012-11-21T03:06:25.000Z","id":"66e11bc6000000000022a834"},{"iso_639_1":"en","iso_3166_1":"US","name":"The Perks of Being a Wallflower - Stephen Chbosky Q&A","key":"gyvUnM3hj_I","site":"YouTube","size":1080,"type":"Featurette","official":true,"published_at":"2012-10-12T18:34:39.000Z","id":"66e11d795142e627648b2bf9"},{"iso_639_1":"en","iso_3166_1":"US","name":"THE PERKS OF BEING A WALLFLOWER - TV Spot \"Master Review\"","key":"eZkoZPEUZG0","site":"YouTube","size":1080,"type":"Teaser","official":true,"published_at":"2012-10-11T01:00:40.000Z","id":"66e11d5e00000000004c9357"},{"iso_639_1":"en","iso_3166_1":"US","name":"The Perks of Being A Wallflower (2012) Official Roundtable \"First Impressions\"","key":"iDjo_jnr4Xs","site":"YouTube","size":1080,"type":"Featurette","official":true,"published_at":"2012-10-05T20:38:01.000Z","id":"66e11d88d7270e37d3afea19"},{"iso_639_1":"en","iso_3166_1":"US","name":"Academy Conversations: The Perks of Being a Wallflower","key":"RffYip5VWbQ","site":"YouTube","size":720,"type":"Featurette","official":true,"published_at":"2012-10-02T22:37:08.000Z","id":"66e11da2f370a2eda254e222"},{"iso_639_1":"en","iso_3166_1":"US","name":"THE PERKS OF BEING A WALLFLOWER - TV Spot \"Review\"","key":"jpM4WApoy78","site":"YouTube","size":1080,"type":"Teaser","official":true,"published_at":"2012-09-20T21:56:18.000Z","id":"66e11cc413d104edad36a725"},{"iso_639_1":"en","iso_3166_1":"US","name":"The Perks Of Being A Wallflower (2012) Official BTS \"Cast & Filmmaker Chat\"","key":"LrUnp17yVM8","site":"YouTube","size":1080,"type":"Behind the Scenes","official":true,"published_at":"2012-06-22T23:24:18.000Z","id":"66e11ccd5a7474fa8c8fcfb9"},{"iso_639_1":"en","iso_3166_1":"US","name":"Trailer","key":"QE7CGX1d6LU","published_at":"2012-06-04T19:11:51.000Z","site":"YouTube","size":1080,"type":"Trailer","official":true,"id":"533ec6d1c3a368544800796d"}]},"external_ids":{"imdb_id":"tt1659337","wikidata_id":"Q675468","facebook_id":"WallflowerMovie","instagram_id":"perksmovie","twitter_id":"WallflowerMovie"}}"#; - -} diff --git a/backend/src/main.rs b/backend/src/main.rs index eb26f35fe..cb247634a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -105,8 +105,23 @@ struct Args { #[arg(long = "dbe")] db_epg_file_name: Option, - #[arg(long = "dbv")] + #[arg(long = "dbv")] // Target Id Mapping db_tim_file_name: Option, + + #[arg(long = "dbms")] // Metadata Retry Status + db_mrs_file_name: Option, +} + +impl Args { + fn db_viewer_args(&self) -> utils::DbViewerArgs<'_> { + utils::DbViewerArgs::new( + self.db_xtream_file_name.as_deref(), + self.db_m3u_file_name.as_deref(), + self.db_epg_file_name.as_deref(), + self.db_tim_file_name.as_deref(), + self.db_mrs_file_name.as_deref(), + ) + } } const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -124,12 +139,7 @@ const BUILD_TIMESTAMP: &str = env!("VERGEN_BUILD_TIMESTAMP"); async fn main() { let args = Args::parse(); - db_viewer( - args.db_xtream_file_name.as_deref(), - args.db_m3u_file_name.as_deref(), - args.db_epg_file_name.as_deref(), - args.db_tim_file_name.as_deref(), - ); + db_viewer(&args.db_viewer_args()); if args.genpwd { match generate_password() { diff --git a/backend/src/model/config/api_user.rs b/backend/src/model/config/api_user.rs index 9ac7123ee..ef6a33cf3 100644 --- a/backend/src/model/config/api_user.rs +++ b/backend/src/model/config/api_user.rs @@ -23,6 +23,7 @@ pub struct ProxyUserCredentials { pub status: Option, pub ui_enabled: bool, pub comment: Option, + pub t_is_api_user: bool, } macros::from_impl!(ProxyUserCredentials); @@ -42,6 +43,7 @@ impl From<&ProxyUserCredentialsDto> for ProxyUserCredentials { status: dto.status, ui_enabled: dto.ui_enabled, comment: dto.comment.clone(), + t_is_api_user: false, } } } diff --git a/backend/src/model/config/metadata_update.rs b/backend/src/model/config/metadata_update.rs index 78da1c9f8..cc8de48aa 100644 --- a/backend/src/model/config/metadata_update.rs +++ b/backend/src/model/config/metadata_update.rs @@ -2,17 +2,7 @@ use crate::model::macros; use shared::model::{ FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, TmdbConfigDto, }; -use shared::utils::{ - default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, - default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, - default_metadata_max_resolve_retry_backoff, default_metadata_probe_cooldown, - default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, - default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, - default_metadata_progress_log_interval, default_metadata_queue_log_interval, - default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, - default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_tmdb_cache_duration_days, - default_tmdb_language, default_tmdb_rate_limit_ms, parse_duration_seconds, parse_size_base_2, -}; +use shared::utils::{default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_max_resolve_retry_backoff, default_metadata_probe_cooldown, default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_match_threshold, default_tmdb_rate_limit_ms, parse_duration_seconds, parse_size_base_2}; #[derive(Debug, Clone)] pub struct MetadataUpdateConfig { @@ -27,6 +17,7 @@ pub struct MetadataUpdateConfig { pub worker_idle_timeout: String, pub worker_idle_timeout_secs: u64, pub max_queue_size: usize, + pub no_change_cache_ttl_secs: u64, } #[derive(Debug, Clone)] @@ -87,6 +78,7 @@ pub struct TmdbConfig { pub language: String, pub cooldown: String, pub cooldown_secs: u64, + pub match_threshold: u16, } impl Default for TmdbConfig { @@ -100,6 +92,7 @@ impl Default for TmdbConfig { language: default_tmdb_language(), cooldown_secs: parse_duration_or_default(&cooldown, &default_metadata_tmdb_cooldown(), false), cooldown, + match_threshold: default_tmdb_match_threshold(), } } } @@ -155,6 +148,7 @@ impl From<&MetadataUpdateConfigDto> for MetadataUpdateConfig { ), worker_idle_timeout: normalized.worker_idle_timeout, max_queue_size: normalized.max_queue_size.max(1), + no_change_cache_ttl_secs: normalized.no_change_cache_ttl_secs.max(1), } } } @@ -171,6 +165,7 @@ impl From<&MetadataUpdateConfig> for MetadataUpdateConfigDto { retry_delay: instance.retry_delay.clone(), worker_idle_timeout: instance.worker_idle_timeout.clone(), max_queue_size: instance.max_queue_size, + no_change_cache_ttl_secs: instance.no_change_cache_ttl_secs, } } } @@ -342,6 +337,7 @@ impl From<&TmdbConfigDto> for TmdbConfig { language: dto.language.clone(), cooldown_secs: parse_duration_or_default(&dto.cooldown, &default_metadata_tmdb_cooldown(), false), cooldown: dto.cooldown.clone(), + match_threshold: dto.match_threshold, } } } @@ -355,6 +351,7 @@ impl From<&TmdbConfig> for TmdbConfigDto { cache_duration_days: instance.cache_duration_days, language: instance.language.clone(), cooldown: instance.cooldown.clone(), + match_threshold: instance.match_threshold, } } } diff --git a/backend/src/model/xtream.rs b/backend/src/model/xtream.rs index bb2cdc60c..93e8389e2 100644 --- a/backend/src/model/xtream.rs +++ b/backend/src/model/xtream.rs @@ -7,7 +7,7 @@ use shared::model::{ PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, ProxyUserStatus, XtreamMappingFlags, XtreamMappingFlagsSet, XtreamMappingOptions, }; -use shared::utils::{arc_str_serde, deserialize_number_from_string_or_zero}; +use shared::utils::{arc_str_serde, concat_path_leading_slash, deserialize_number_from_string_or_zero}; use enum_iterator::all; use crate::model::XtreamTargetFlags; @@ -36,8 +36,8 @@ impl XtreamCategory { pub fn xtream_mapping_option_from_target_options(target: &ConfigTarget, target_output: &XtreamTargetOutput, - cfg: &AppConfig, user: &ProxyUserCredentials, - resource_url: Option<&str>) -> XtreamMappingOptions { + app_config: &AppConfig, user: &ProxyUserCredentials, + encrypt_secret: [u8; 16]) -> XtreamMappingOptions { let force_redirect = target.options.as_ref().and_then(|o| o.force_redirect); let mut reverse_item_types = PlaylistItemTypeSet::empty(); @@ -67,17 +67,33 @@ pub fn xtream_mapping_option_from_target_options(target: &ConfigTarget, target_o { flags.set(XtreamMappingFlags::SkipSeriesDirectSource); } - if cfg.is_reverse_proxy_resource_rewrite_enabled() { + if app_config.is_reverse_proxy_resource_rewrite_enabled() { flags.set(XtreamMappingFlags::RewriteResourceUrl); } + + let base_url = if user.t_is_api_user { + let config = app_config.config.load(); + let web_ui_path = config + .web_ui + .as_ref() + .and_then(|w| w.path.as_ref()) + .map_or("", String::as_str); + concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource") + } else { + app_config.get_user_server_info(user).get_base_url() + }; + + XtreamMappingOptions { flags, force_redirect, reverse_item_types, username: user.username.clone(), password: user.password.clone(), - base_url: resource_url.map(ToString::to_string), + base_url, + web_ui_request: user.t_is_api_user, + encrypt_secret } } diff --git a/backend/src/processing/parser/hls.rs b/backend/src/processing/parser/hls.rs index a5abd9e68..edca58d8f 100644 --- a/backend/src/processing/parser/hls.rs +++ b/backend/src/processing/parser/hls.rs @@ -1,22 +1,27 @@ -use std::borrow::Cow; use crate::model::ProxyUserCredentials; -use crate::utils::{deobfuscate_text, obfuscate_text}; -use shared::utils::{extract_extension_from_url, CONSTANTS, HLS_PREFIX}; +use shared::concat_string; +use shared::utils::{deobfuscate_text, extract_extension_from_url, obfuscate_text, CONSTANTS, HLS_PREFIX}; +use std::borrow::Cow; use std::str; use url::Url; -use shared::concat_string; const TOKEN_SEPARATOR: char = '\x1F'; const TOKEN_SEPARATOR_STR: &str = "\x1F"; -fn create_hls_session_token_and_url(secret: &[u8], session_token: &str, stream_url: &str) -> Option { - if let Ok(cookie_value) = obfuscate_text(secret, &concat_string!(session_token, TOKEN_SEPARATOR_STR, stream_url)) { - if let Some(ext) = extract_extension_from_url(stream_url) { - return Some(concat_string!(&cookie_value, &ext)); - } - return Some(cookie_value); +fn create_hls_session_token_and_url(secret: &[u8], session_token: &str, stream_url: &str) -> String { + let cookie_value = obfuscate_text(secret, &concat_string!(session_token, TOKEN_SEPARATOR_STR, stream_url)); + if let Some(ext) = extract_extension_from_url(stream_url) { + return concat_string!(&cookie_value, &ext); } - None + cookie_value +} + +fn create_hls_url_without_session_token(secret: &[u8], stream_url: &str) -> String { + let token = obfuscate_text(secret, stream_url); + if let Some(ext) = extract_extension_from_url(stream_url) { + return concat_string!(&token, &ext); + } + token } fn remove_any_ext(s: &str) -> &str { @@ -33,6 +38,9 @@ pub fn get_hls_session_token_and_url_from_token(secret: &[u8], token: &str) -> O let stream_url: String = parts[1].to_string(); return Some((Some(session_token), stream_url)); } + if parts.len() == 1 { + return Some((None, decrypted)); + } } None } @@ -49,7 +57,7 @@ pub struct RewriteHlsProps<'a> { /// Rewrites an HLS URI relative to a base playlist URL. /// Absolute URIs are returned unchanged. -pub fn rewrite_hls_url<'a>(base: &'a str, reference: &'a str) -> Cow<'a, str> { +pub fn rewrite_hls_url<'a>(base: &'a str, reference: &'a str) -> Cow<'a, str> { // absolute URI → passthrough if Url::parse(reference).is_ok() { return Cow::Borrowed(reference); @@ -71,11 +79,11 @@ fn rewrite_uri_attrib<'a>(line: &'a str, props: &RewriteHlsProps) -> Cow<'a, str let rewritten = rewrite_hls_url(&props.hls_url, uri); let final_uri = if let Some(user_token) = &props.user_token { - create_hls_session_token_and_url( + Cow::Owned(create_hls_session_token_and_url( props.secret, user_token, &rewritten, - ).map(Cow::Owned).unwrap_or(rewritten) + )) } else { rewritten }; @@ -104,20 +112,21 @@ pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> Stri // target url let target_url = rewrite_hls_url(&props.hls_url, line); - if let Some(user_token) = &props.user_token { - if let Some(token) = create_hls_session_token_and_url(props.secret, user_token, &target_url) { - let url = format!( - "{}/{HLS_PREFIX}/{}/{}/{}/{}/{}", - props.base_url, - username, - password, - props.input_id, - props.virtual_id, - token - ); - result.push(url); - } - } + let token = if let Some(user_token) = &props.user_token { + create_hls_session_token_and_url(props.secret, user_token, &target_url) + } else { + create_hls_url_without_session_token(props.secret, &target_url) + }; + let url = format!( + "{}/{HLS_PREFIX}/{}/{}/{}/{}/{}", + props.base_url, + username, + password, + props.input_id, + props.virtual_id, + token + ); + result.push(url); } result.push("\r\n".to_string()); result.join("\r\n") @@ -125,9 +134,12 @@ pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> Stri #[cfg(test)] mod test { + use crate::model::ProxyUserCredentials; + use crate::processing::parser::hls::{ + get_hls_session_token_and_url_from_token, rewrite_hls, rewrite_hls_url, RewriteHlsProps, + }; use rand::RngCore; - use shared::utils::u32_to_base64; - use crate::processing::parser::hls::{rewrite_hls_url}; + use shared::utils::{u32_to_base64, HLS_PREFIX}; #[test] fn test_token_size() { @@ -208,4 +220,36 @@ mod test { let out = rewrite_hls_url(base, fragment); assert_eq!(out, "http://example.com/hls/seg.ts#t=10"); } -} \ No newline at end of file + + #[test] + fn rewrite_hls_without_user_token_keeps_segment_urls() { + let mut user = ProxyUserCredentials::default(); + user.username = "u".to_string(); + user.password = "p".to_string(); + let secret = [7u8; 16]; + let props = RewriteHlsProps { + secret: &secret, + base_url: "http://proxy", + content: "#EXTM3U\nsegment.ts", + hls_url: "http://origin/live/main.m3u8".to_string(), + virtual_id: 101, + input_id: 11, + user_token: None, + }; + + let rewritten = rewrite_hls(&user, &props); + let segment_line = rewritten + .lines() + .find(|line| line.contains(&format!("/{HLS_PREFIX}/"))) + .expect("rewritten playlist should contain a segment URL"); + let token = segment_line + .rsplit('/') + .next() + .expect("rewritten hls segment URL should include token"); + let decoded = get_hls_session_token_and_url_from_token(&secret, token) + .expect("rewritten hls token should decode"); + + assert!(decoded.0.is_none()); + assert_eq!(decoded.1, "http://origin/live/segment.ts"); + } +} diff --git a/backend/src/processing/processor/mod.rs b/backend/src/processing/processor/mod.rs index 243cbfbb7..344242733 100644 --- a/backend/src/processing/processor/mod.rs +++ b/backend/src/processing/processor/mod.rs @@ -14,7 +14,9 @@ pub use self::xtream::*; pub use self::xtream_vod::*; pub use self::xtream_series::*; pub use self::stream_probe::*; +use crate::api::model::ProviderHandle; use shared::create_bitset; +use tokio_util::sync::CancellationToken; create_bitset!(u8, ResolveOptionsFlags, Resolve, TmdbMissing, Probe, Background); @@ -47,6 +49,15 @@ pub(crate) const FOREGROUND_BATCH_SIZE: usize = 200; pub(crate) const FOREGROUND_RETRY_BATCH_MAX_SIZE: usize = FOREGROUND_BATCH_SIZE * 4; pub(crate) const FOREGROUND_MIN_RETRY_DELAY_SECS: u64 = 1; +pub(crate) fn select_cancel_token<'a>( + acquired_handle: Option<&'a ProviderHandle>, + active_handle: Option<&'a ProviderHandle>, +) -> Option<&'a CancellationToken> { + acquired_handle + .and_then(|h| h.cancel_token.as_ref()) + .or_else(|| active_handle.and_then(|h| h.cancel_token.as_ref())) +} + // // fn get_resolve__options(target: &ConfigTarget, fpl: &FetchedPlaylist) -> bool diff --git a/backend/src/processing/processor/stream_probe.rs b/backend/src/processing/processor/stream_probe.rs index 272517488..36506c15e 100644 --- a/backend/src/processing/processor/stream_probe.rs +++ b/backend/src/processing/processor/stream_probe.rs @@ -1,6 +1,7 @@ use crate::api::model::ActiveProviderManager; use crate::model::ConfigInput; use crate::model::{AppConfig}; +use crate::processing::processor::select_cancel_token; use crate::repository::{get_input_m3u_playlist_file_path, get_input_storage_path, get_input_local_library_playlist_file_path, xtream_get_file_path, BPlusTreeUpdate}; use crate::utils::{debug_if_enabled, ffmpeg}; use crate::utils::ffmpeg::{ProbeFailureKind, ProbeUrlOutcome}; @@ -115,14 +116,17 @@ pub async fn update_generic_stream_metadata( debug_if_enabled!("Probing Generic Stream '{unique_id}'"); - let probe_data = ffmpeg::probe_url( + let cancel_token = select_cancel_token(acquired_handle.as_ref(), active_handle); + let probe_data = ffmpeg::probe_url_with_cancel( &probe_url, user_agent.as_deref(), analyze_duration, probe_size, ffprobe_timeout, config.proxy.as_ref(), - ).await; + cancel_token, + ) + .await; if let Some(handle) = acquired_handle { active_provider.release_handle(&handle).await; @@ -138,6 +142,10 @@ pub async fn update_generic_stream_metadata( warn!("Probe failed or timed out for generic stream: {unique_id}"); return Ok(GenericProbeOutcome::ProbeFailed); } + ProbeUrlOutcome::Failed(ProbeFailureKind::Cancelled) => { + warn!("Probe cancelled for generic stream: {unique_id}"); + return Ok(GenericProbeOutcome::ProbeFailed); + } }; // Hold the async file lock while the blocking DB update runs in a blocking thread. diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index 06ceb411e..8e6862697 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -164,6 +164,10 @@ pub async fn update_live_stream_metadata( warn!("Probe failed for Live Stream ID {} (Input: {})", display_id, input.name); // We still persist the updated last_probed_timestamp so we don't retry immediately } + ProbeUrlOutcome::Failed(ProbeFailureKind::Cancelled) => { + warn!("Probe cancelled for Live Stream ID {} (Input: {})", display_id, input.name); + // We still persist the updated last_probed_timestamp so we don't retry immediately + } } // 4. Persist diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 0a5a32c11..0d1903289 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -8,8 +8,8 @@ use crate::processing::parser::xtream::create_xtream_series_episode_url; use crate::processing::parser::xtream::parse_xtream_series_info; use crate::processing::processor::playlist::{PlaylistProcessingContext, ProcessingPipe}; use crate::processing::processor::{ - create_resolve_options_function_for_xtream_target, process_foreground_retry_once, ResolveOptions, - ResolveOptionsFlags, FOREGROUND_BATCH_SIZE as BATCH_SIZE, FOREGROUND_MIN_RETRY_DELAY_SECS, + create_resolve_options_function_for_xtream_target, process_foreground_retry_once, select_cancel_token, + ResolveOptions, ResolveOptionsFlags, FOREGROUND_BATCH_SIZE as BATCH_SIZE, FOREGROUND_MIN_RETRY_DELAY_SECS, FOREGROUND_RETRY_BATCH_MAX_SIZE as RETRY_BATCH_MAX_SIZE, }; use crate::ptt::ptt_parse_title; @@ -30,6 +30,7 @@ use shared::model::{ }; use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster}; use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -524,6 +525,7 @@ async fn update_series_info_immediate( reasons.contains(ResolveReason::Probe), probe_settings, db_query, + None, ) .await } @@ -620,6 +622,7 @@ pub async fn update_series_metadata( do_probe: bool, probe_settings: SeriesProbeSettings, db_query: Option>>>, + tmdb_and_date_present_out: Option<&AtomicBool>, ) -> Result, TuliproxError> { let working_dir = &app_config.config.load().working_dir; let storage_path = get_input_storage_path(&input.name, working_dir) @@ -818,6 +821,13 @@ pub async fn update_series_metadata( } } + // Report whether TMDB and release date are present (regardless of whether we updated them this call). + if let Some(out) = tmdb_and_date_present_out { + let has_tmdb = properties.tmdb.is_some(); + let has_date = properties.release_date.is_some(); + out.store(has_tmdb && has_date, Ordering::Relaxed); + } + // 3. Probe Episodes (if enabled) if do_probe && app_config.is_ffprobe_enabled().await { if let Some(details) = properties.details.as_mut() { @@ -863,13 +873,15 @@ pub async fn update_series_metadata( ep.title, ep.season, ep.episode_num, missing_reason ); - match crate::utils::ffmpeg::probe_url( + let cancel_token = select_cancel_token(temp_handle.as_ref(), active_handle); + match crate::utils::ffmpeg::probe_url_with_cancel( &episode_url, user_agent.as_deref(), probe_settings.analyze_duration_micros, probe_settings.probe_size_bytes, probe_settings.timeout_secs, config.proxy.as_ref(), + cancel_token, ) .await { @@ -892,6 +904,11 @@ pub async fn update_series_metadata( probe_failure = Some(ProbeFailureKind::Other); } } + ProbeUrlOutcome::Failed(ProbeFailureKind::Cancelled) => { + if probe_failure.is_none() { + probe_failure = Some(ProbeFailureKind::Cancelled); + } + } } if let Some(h) = temp_handle { @@ -923,6 +940,7 @@ pub async fn update_series_metadata( shared::error::info_err!("Probe failed with 404 Not Found for Series {display_id}") } ProbeFailureKind::Other => shared::error::info_err!("Probe failed for Series {display_id}"), + ProbeFailureKind::Cancelled => shared::error::info_err!("Probe cancelled for Series {display_id}"), }; return Err(err); } diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index ef0cc0052..638b5d6a2 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -7,8 +7,8 @@ use crate::model::{AppConfig, ConfigTarget}; use crate::model::{ConfigInput, ConfigInputFlags}; use crate::processing::processor::playlist::PlaylistProcessingContext; use crate::processing::processor::{ - create_resolve_options_function_for_xtream_target, process_foreground_retry_once, ResolveOptions, - ResolveOptionsFlags, FOREGROUND_BATCH_SIZE as BATCH_SIZE, FOREGROUND_MIN_RETRY_DELAY_SECS, + create_resolve_options_function_for_xtream_target, process_foreground_retry_once, select_cancel_token, + ResolveOptions, ResolveOptionsFlags, FOREGROUND_BATCH_SIZE as BATCH_SIZE, FOREGROUND_MIN_RETRY_DELAY_SECS, FOREGROUND_RETRY_BATCH_MAX_SIZE as RETRY_BATCH_MAX_SIZE, }; use crate::ptt::ptt_parse_title; @@ -27,6 +27,7 @@ use shared::model::{ VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo, }; use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -479,6 +480,7 @@ async fn update_vod_info_immediate( resolve_tmdb, reasons.contains(ResolveReason::Probe), db_query, + None, ) .await } @@ -505,6 +507,7 @@ pub async fn update_vod_metadata( resolve_tmdb: bool, do_probe: bool, db_query: Option>>>, + tmdb_resolved_out: Option<&AtomicBool>, ) -> Result, TuliproxError> { let working_dir = &app_config.config.load().working_dir; let storage_path = get_input_storage_path(&input.name, working_dir) @@ -747,6 +750,17 @@ pub async fn update_vod_metadata( } } + // Report whether TMDB data is present (regardless of whether we updated it this call). + if let Some(out) = tmdb_resolved_out { + let has_tmdb = properties.tmdb.is_some(); + let has_date = properties + .details + .as_ref() + .and_then(|d| d.release_date.as_ref()) + .is_some(); + out.store(has_tmdb && has_date, Ordering::Relaxed); + } + // 3. Probe (if enabled globally in config) let ffprobe_enabled = app_config.is_ffprobe_enabled().await; if do_probe && ffprobe_enabled { @@ -793,13 +807,15 @@ pub async fn update_vod_metadata( if active_handle.is_some() || temp_handle.is_some() { debug_if_enabled!("Probing VOD '{}' (ID: {})", display_title, display_id); - match crate::utils::ffmpeg::probe_url( + let cancel_token = select_cancel_token(temp_handle.as_ref(), active_handle); + match crate::utils::ffmpeg::probe_url_with_cancel( &stream_url, user_agent.as_deref(), analyze_duration, probe_size, ffprobe_timeout, config.proxy.as_ref(), + cancel_token, ) .await { @@ -823,6 +839,11 @@ pub async fn update_vod_metadata( probe_failure = Some(ProbeFailureKind::Other); } } + ProbeUrlOutcome::Failed(ProbeFailureKind::Cancelled) => { + if probe_failure.is_none() { + probe_failure = Some(ProbeFailureKind::Cancelled); + } + } } if let Some(h) = temp_handle { active_provider.release_handle(&h).await; @@ -841,6 +862,7 @@ pub async fn update_vod_metadata( shared::error::info_err!("Probe failed with 404 Not Found for VOD {display_id}") } ProbeFailureKind::Other => shared::error::info_err!("Probe failed for VOD {display_id}"), + ProbeFailureKind::Cancelled => shared::error::info_err!("Probe cancelled for VOD {display_id}"), }; return Err(err); } diff --git a/backend/src/repository/library_repository.rs b/backend/src/repository/library_repository.rs index 26e328d73..81adfa460 100644 --- a/backend/src/repository/library_repository.rs +++ b/backend/src/repository/library_repository.rs @@ -105,7 +105,6 @@ fn preserve_local_probe_state_if_unchanged(new_item: &mut XtreamPlaylistItem, ol } } - pub async fn load_input_local_library_playlist(app_config: &Arc, lib_path: &Path) -> Result, TuliproxError> { if file_exists_async(lib_path).await { let file_lock = app_config.file_locks.read_lock(lib_path).await; diff --git a/backend/src/repository/user_repository.rs b/backend/src/repository/user_repository.rs index e06abe2e3..ff29f42ff 100644 --- a/backend/src/repository/user_repository.rs +++ b/backend/src/repository/user_repository.rs @@ -47,6 +47,7 @@ impl StoredProxyUserCredentialsDeprecated { status: stored.status, ui_enabled: stored.ui_enabled, comment: stored.comment.clone(), + t_is_api_user: false, } } } @@ -108,6 +109,7 @@ impl StoredProxyUserCredentials { status: stored.status, ui_enabled: stored.ui_enabled, comment: stored.comment.clone(), + t_is_api_user: false, } } } @@ -456,6 +458,7 @@ mod tests { status: Some(ProxyUserStatus::Active), ui_enabled: true, comment: None, + t_is_api_user: false, }, ProxyUserCredentials { username: "Test2".to_string(), @@ -471,6 +474,7 @@ mod tests { status: Some(ProxyUserStatus::Expired), ui_enabled: true, comment: None, + t_is_api_user: false, }, ProxyUserCredentials { username: "Test3".to_string(), @@ -486,6 +490,7 @@ mod tests { status: Some(ProxyUserStatus::Expired), ui_enabled: true, comment: None, + t_is_api_user: false, }, ProxyUserCredentials { username: "Test4".to_string(), @@ -501,6 +506,7 @@ mod tests { status: Some(ProxyUserStatus::Expired), ui_enabled: true, comment: None, + t_is_api_user: false, } ], }; diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index bcb4d4008..1dcff034a 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -10,9 +10,11 @@ use shared::model::{PlaylistItemType, TargetType, XtreamCluster, XtreamMappingOp use std::collections::HashSet; use crate::repository::get_file_path_for_db_index; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use tokio::sync::mpsc; use tokio::task; +use crate::api::model::AppState; pub struct XtreamPlaylistIterator { inner: LockedReceiverStream<(XtreamPlaylistItem, bool)>, @@ -150,22 +152,22 @@ pub struct XtreamPlaylistJsonIterator { impl XtreamPlaylistJsonIterator { pub async fn new( cluster: XtreamCluster, - config: &AppConfig, + app_state: &Arc, target: &ConfigTarget, category_id: Option, user: &ProxyUserCredentials, ) -> Result { let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected: xtream output required for target {}", target.name))?; - let server_info = config.get_user_server_info(user); + let encrypt_secret = app_state.get_encrypt_secret(); let options = xtream_mapping_option_from_target_options( target, xtream_output, - config, + &app_state.app_config, user, - Some(server_info.get_base_url().as_str()), + encrypt_secret ); Ok(Self { - inner: XtreamPlaylistIterator::new(cluster, config, target, category_id, user).await?, + inner: XtreamPlaylistIterator::new(cluster, &app_state.app_config, target, category_id, user).await?, options, }) } diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index aa1494aee..b899d5da9 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -637,12 +637,12 @@ pub async fn xtream_get_item_for_stream_id( pub async fn xtream_load_rewrite_playlist( cluster: XtreamCluster, - config: &AppConfig, + app_state: &Arc, target: &ConfigTarget, category_id: Option, user: &ProxyUserCredentials, ) -> Result { - XtreamPlaylistJsonIterator::new(cluster, config, target, category_id, user).await + XtreamPlaylistJsonIterator::new(cluster, app_state, target, category_id, user).await } pub async fn iter_raw_xtream_target_playlist(app_config: &AppConfig, target: &ConfigTarget, cluster: XtreamCluster) -> Option + Send + Unpin>> { diff --git a/backend/src/utils/crypto_utils.rs b/backend/src/utils/crypto_utils.rs index b0b8f4ec2..eab5b723f 100644 --- a/backend/src/utils/crypto_utils.rs +++ b/backend/src/utils/crypto_utils.rs @@ -2,40 +2,13 @@ use base64::{engine::general_purpose, Engine as _}; use openssl::symm::{Cipher, Crypter, Mode}; use rand::{RngCore, rngs::OsRng, TryRngCore}; use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::utils::encode_base64_string; pub fn encode_base64_hash(text: &str) -> String { let hash = blake3::hash(text.as_bytes()); encode_base64_string(hash.as_bytes()) } -pub fn encode_base64_string(input: &[u8]) -> String { - general_purpose::URL_SAFE_NO_PAD.encode(input) -} - -pub fn decode_base64_string(input: &str) -> Vec { - general_purpose::URL_SAFE_NO_PAD.decode(input).unwrap_or_else(|_| input.as_bytes().to_vec()) -} - -pub fn xor_bytes(secret: &[u8], data: &[u8]) -> Vec { - data.iter() - .enumerate() - .map(|(i, &b)| b ^ secret[i % secret.len()]) - .collect() -} - -pub fn obfuscate_text(secret: &[u8], text: &str) -> Result { - Ok(encode_base64_string(&xor_bytes(secret, text.as_bytes()))) -} - -pub fn deobfuscate_text(secret: &[u8], text: &str) -> Result { - let data = xor_bytes(secret, &decode_base64_string(text)); - if let Ok(result) = String::from_utf8(data) { - Ok(result) - } else { - Err(text.to_string()) - } -} - pub fn obscure_text(secret: &[u8;16], url: &str) -> Result { let mut iv = [0u8; 16]; if OsRng.try_fill_bytes(&mut iv).is_err() { @@ -84,7 +57,7 @@ pub fn deobscure_text(secret: &[u8;16], encoded: &str) -> Result bool; + +struct DumpRequest<'a> { + filename: Option<&'a str>, + label: &'static str, + dump_fn: DumpFn, } -pub fn db_viewer(xtream_filename: Option<&str>, - m3u_filename: Option<&str>, - epg_filename: Option<&str>, - tim_filename: Option<&str>,) { +pub struct DbViewerArgs<'a> { + pub xtream_filename: Option<&'a str>, + pub m3u_filename: Option<&'a str>, + pub epg_filename: Option<&'a str>, + pub tim_filename: Option<&'a str>, + pub metadata_status_filename: Option<&'a str>, +} + +impl<'a> DbViewerArgs<'a> { + pub const fn new( + xtream_filename: Option<&'a str>, + m3u_filename: Option<&'a str>, + epg_filename: Option<&'a str>, + tim_filename: Option<&'a str>, + metadata_status_filename: Option<&'a str>, + ) -> Self { + Self { + xtream_filename, + m3u_filename, + epg_filename, + tim_filename, + metadata_status_filename, + } + } +} + +pub fn db_viewer(args: &DbViewerArgs<'_>) { + let requests = [ + DumpRequest { + filename: args.xtream_filename, + label: "xtream", + dump_fn: dump_xtream_db, + }, + DumpRequest { + filename: args.m3u_filename, + label: "m3u", + dump_fn: dump_m3u_db, + }, + DumpRequest { + filename: args.epg_filename, + label: "epg", + dump_fn: dump_epg_db, + }, + DumpRequest { + filename: args.tim_filename, + label: "target_id_mapping", + dump_fn: dump_target_mapping_db, + }, + DumpRequest { + filename: args.metadata_status_filename, + label: "metadata_status", + dump_fn: dump_metadata_status_db, + }, + ]; + + let any_requested = requests.iter().any(|request| request.filename.is_some()); + if !any_requested { + return; + } + + init_db_viewer_logger(); + let mut any_processed = false; - if let Some(filename) = xtream_filename { - any_processed = true; - if !dump_db(filename, DbType::Xtream) { - exit_app(1); - } - } - if let Some(filename) = m3u_filename { - any_processed = true; - if !dump_db(filename, DbType::M3u) { - exit_app(1); - } - } - - if let Some(filename) = epg_filename { - any_processed = true; - if !dump_db(filename, DbType::Epg) { - exit_app(1); - } - } - - if let Some(filename) = tim_filename { - any_processed = true; - if !dump_db(filename, DbType::TargetIdMapping) { - exit_app(1); + for request in requests { + if let Some(filename) = request.filename { + any_processed = true; + if !dump_db(filename, request.label, request.dump_fn) { + exit_app(1); + } } } if any_processed { - exit_app(1); + exit_app(0); } } -fn dump_db(filename: &str, db_type: DbType) -> bool { +fn init_db_viewer_logger() { let mut log_builder = Builder::from_default_env(); log_builder.target(Target::Stderr); log_builder.filter_level(LevelFilter::Info); let _ = log_builder.try_init(); +} +fn try_dump_typed_db(path: &Path) -> bool +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + if let Ok(mut query) = BPlusTreeQuery::::try_new(path) { + return print_json_from_iter(query.iter()); + } + false +} + +fn try_dump_m3u_with_key(path: &Path) -> Result +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut query = BPlusTreeQuery::::try_new(path).map_err(|err| err.to_string())?; + query.len().map_err(|err| err.to_string())?; + Ok(print_json_from_iter(query.iter())) +} + +fn dump_xtream_db(path: &Path) -> bool { try_dump_typed_db::(path) } + +fn dump_m3u_db(path: &Path) -> bool { + // M3U DB keys can be u32 (target playlists) or Arc (input playlists). + let err_u32 = match try_dump_m3u_with_key::(path) { + Ok(result) => return result, + Err(err) => Some(err), + }; + let err_str = match try_dump_m3u_with_key::>(path) { + Ok(result) => return result, + Err(err) => Some(err), + }; + + error!( + "Failed to open M3U DB with any known key type at {}: u32_err={:?}, string_err={:?}", + path.display(), + err_u32, + err_str + ); + false +} + +fn dump_epg_db(path: &Path) -> bool { try_dump_typed_db::, EpgChannel>(path) } + +fn dump_target_mapping_db(path: &Path) -> bool { try_dump_typed_db::(path) } + +fn dump_metadata_status_db(path: &Path) -> bool { + try_dump_typed_db::(path) +} + +fn dump_db(filename: &str, label: &str, dump_fn: DumpFn) -> bool { match PathBuf::from(filename).canonicalize() { Ok(path) => { - match db_type { - DbType::Xtream => { - if let Ok(mut query) = BPlusTreeQuery::::try_new(&path) { - let iterator = query.iter(); - return print_json_from_iter(iterator); - } - } - DbType::M3u => { - // M3U DB keys can be u32 (target playlists) or Arc (input playlists). - let err_u32 = match BPlusTreeQuery::::try_new(&path) { - Ok(mut query) => match query.len() { - Ok(_) => { - let iterator = query.iter(); - return print_json_from_iter(iterator); - } - Err(err) => Some(err.to_string()), - }, - Err(err) => Some(err.to_string()), - }; - - let err_str = match BPlusTreeQuery::, M3uPlaylistItem>::try_new(&path) { - Ok(mut query) => match query.len() { - Ok(_) => { - let iterator = query.iter(); - return print_json_from_iter(iterator); - } - Err(err) => Some(err.to_string()), - }, - Err(err) => Some(err.to_string()), - }; - - error!( - "Failed to open M3U DB with any known key type at {}: u32_err={:?}, string_err={:?}", - path.display(), - err_u32, - err_str - ); - } - DbType::Epg => { - if let Ok(mut query) = BPlusTreeQuery::, EpgChannel>::try_new(&path) { - let iterator = query.iter(); - return print_json_from_iter(iterator); - } - } - DbType::TargetIdMapping => { - if let Ok(mut query) = BPlusTreeQuery::::try_new(&path) { - let iterator = query.iter(); - return print_json_from_iter(iterator); - } - } + if !dump_fn(&path) { + error!("Failed to dump {label} DB at {}", path.display()); + return false; } + true } Err(err) => { - error!("Invalid file path! {err}"); + error!("Invalid file path for {label} DB: {err}"); + false } } - - false } fn print_json_from_iter(iterator: BPlusTreeDiskIterator) -> bool @@ -146,7 +192,7 @@ where } println!("]"); - error_count <= 0 + error_count == 0 } fn exit_app(code: i32) { diff --git a/backend/src/utils/epg_parser.rs b/backend/src/utils/epg_parser.rs index 9d9caff5f..93a721659 100644 --- a/backend/src/utils/epg_parser.rs +++ b/backend/src/utils/epg_parser.rs @@ -55,7 +55,7 @@ pub struct EpgProcessingOptions { pub fn get_epg_processing_options(app_state: &Arc, user: &ProxyUserCredentials, target: &Arc) -> EpgProcessingOptions { let rewrite_resources = app_state.app_config.is_reverse_proxy_resource_rewrite_enabled(); - let encrypt_secret = app_state.app_config.get_reverse_proxy_rewrite_secret().unwrap_or_else(|| app_state.app_config.encrypt_secret); + let encrypt_secret = app_state.get_encrypt_secret(); // If redirect is true → rewrite_urls = false → keep original // If redirect is false and rewrite_resources is true → rewrite_urls = true → rewriting allowed diff --git a/backend/src/utils/ffmpeg.rs b/backend/src/utils/ffmpeg.rs index 11678d70b..72fec0d5d 100644 --- a/backend/src/utils/ffmpeg.rs +++ b/backend/src/utils/ffmpeg.rs @@ -10,6 +10,7 @@ use url::Url; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProbeFailureKind { NotFound, + Cancelled, Other, } @@ -117,55 +118,47 @@ pub async fn probe_url( return ProbeUrlOutcome::Failed(ProbeFailureKind::Other); } - let stdout = String::from_utf8_lossy(&output.stdout); - if let Ok(json) = serde_json::from_str::(&stdout) { - let streams = json.get("streams").and_then(|s| s.as_array()); + if let Ok(json) = serde_json::from_slice::(&output.stdout) { + if let Some(stream_list) = json.get("streams").and_then(Value::as_array) { + // Single-pass stream detection: prefer codec_type, fall back to structural hints. + let mut video_stream: Option<&Value> = None; + let mut audio_stream: Option<&Value> = None; - if let Some(stream_list) = streams { - let mut video_info: Option = None; - let mut audio_info: Option = None; - let mut raw_video_json: Option = None; - let mut raw_audio_json: Option = None; + for stream in stream_list { + let codec_type = stream.get("codec_type").and_then(Value::as_str); + if video_stream.is_none() + && (codec_type == Some("video") + || (codec_type.is_none() + && (stream.get("width").is_some() || stream.get("height").is_some()))) + { + video_stream = Some(stream); + } else if audio_stream.is_none() + && (codec_type == Some("audio") + || (codec_type.is_none() + && (stream.get("channels").is_some() + || stream.get("channel_layout").is_some()))) + { + audio_stream = Some(stream); + } + if video_stream.is_some() && audio_stream.is_some() { + break; + } + } - for stream in stream_list { - // Check codec_type if available - let codec_type = stream.get("codec_type").and_then(|s| s.as_str()); - - // We prefer the first video/audio stream we find - if codec_type == Some("video") && video_info.is_none() { - video_info = Some(stream.to_string()); - raw_video_json = Some(stream.clone()); - } else if codec_type == Some("audio") && audio_info.is_none() { - audio_info = Some(stream.to_string()); - raw_audio_json = Some(stream.clone()); - } - } - - // Fallback heuristic if codec_type missing - if video_info.is_none() { - for stream in stream_list { - if (stream.get("width").is_some() || stream.get("height").is_some()) && video_info.is_none() { - video_info = Some(stream.to_string()); - raw_video_json = Some(stream.clone()); - } - } - } - if audio_info.is_none() { - for stream in stream_list { - if (stream.get("channels").is_some() || stream.get("channel_layout").is_some()) && audio_info.is_none() { - audio_info = Some(stream.to_string()); - raw_audio_json = Some(stream.clone()); - } - } - } - - if video_info.is_some() || audio_info.is_some() { - let mq = MediaQuality::from_ffprobe_info(audio_info.as_deref(), video_info.as_deref()); - if let Some(quality) = mq { - return ProbeUrlOutcome::Success(quality, raw_video_json, raw_audio_json); - } - } - } + if video_stream.is_some() || audio_stream.is_some() { + // Materialize strings only for the selected streams. + let video_str = video_stream.map(Value::to_string); + let audio_str = audio_stream.map(Value::to_string); + let mq = MediaQuality::from_ffprobe_info(audio_str.as_deref(), video_str.as_deref()); + if let Some(quality) = mq { + return ProbeUrlOutcome::Success( + quality, + video_stream.cloned(), + audio_stream.cloned(), + ); + } + } + } } else { warn!("Failed to parse ffprobe json output for {}", sanitize_sensitive_info(url)); } @@ -181,6 +174,32 @@ pub async fn probe_url( ProbeUrlOutcome::Failed(ProbeFailureKind::Other) } +/// Wrapper around [`probe_url`] that races the probe against an optional cancellation token. +/// When the token fires, the probe future is dropped (`kill_on_drop` kills the ffprobe process) +/// and `ProbeUrlOutcome::Failed(Cancelled)` is returned immediately. +pub async fn probe_url_with_cancel( + url: &str, + user_agent: Option<&str>, + analyze_duration: u64, + probe_size: u64, + timeout_secs: u64, + proxy_cfg: Option<&ProxyConfig>, + cancel_token: Option<&tokio_util::sync::CancellationToken>, +) -> ProbeUrlOutcome { + if let Some(token) = cancel_token { + tokio::select! { + biased; + () = token.cancelled() => { + warn!("Probe preempted for {}", shared::utils::sanitize_sensitive_info(url)); + ProbeUrlOutcome::Failed(ProbeFailureKind::Cancelled) + } + result = probe_url(url, user_agent, analyze_duration, probe_size, timeout_secs, proxy_cfg) => result, + } + } else { + probe_url(url, user_agent, analyze_duration, probe_size, timeout_secs, proxy_cfg).await + } +} + #[cfg(test)] mod tests { use super::build_ffprobe_proxy_url; diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index 9c33be430..f3feb8a3a 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -21,7 +21,7 @@ use shared::model::{ ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, MsgKind, PatternTemplate, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, }; -use shared::utils::{CONSTANTS, PROVIDER_SCHEME_PREFIX, TEMPLATE_FILE}; +use shared::utils::{generate_default_access_secret, generate_default_encrypt_secret, CONSTANTS, PROVIDER_SCHEME_PREFIX, TEMPLATE_FILE}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::env; @@ -573,8 +573,8 @@ pub async fn read_initial_app_config( paths: Arc::new(ArcSwap::from_pointee(paths.clone())), file_locks: Arc::new(FileLockManager::default()), custom_stream_response: Arc::new(ArcSwapAny::default()), - access_token_secret: Default::default(), - encrypt_secret: Default::default(), + access_token_secret: generate_default_access_secret(), + encrypt_secret: generate_default_encrypt_secret(), ffprobe_available: Arc::new(OnceCell::new()) }; app_config.prepare(include_computed)?; diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index f43f5c083..d7c42c494 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -75,8 +75,8 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected error, missing xtream output"))?; let app_config = &app_state.app_config; - let server_info = app_config.get_user_server_info(user); - let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str())); + let encrypt_secret = app_state.get_encrypt_secret(); + let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, encrypt_secret); if let Some(content) = pli.get_resolved_info_document(&options) { return serde_json::to_string(&content).map_err(|err| info_err!("{err}")); @@ -245,8 +245,8 @@ fn xtream_resolve_stream_info(app_state: &Arc, user: &ProxyUserCredent target: &ConfigTarget, xtream_output: &XtreamTargetOutput, pli: &XtreamPlaylistItem) -> Option> { let app_config = &app_state.app_config; - let server_info = app_config.get_user_server_info(user); - let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str())); + let encrypt_secret = app_state.get_encrypt_secret(); + let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, encrypt_secret); if let Some(content) = pli.get_resolved_info_document(&options) { return Some(serde_json::to_string(&content).map_err(|err| info_err!("Failed to serialize stream info: {err}"))); } diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 2ccd804e1..e4c9b2577 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -582,7 +582,8 @@ "COOLDOWN": "Pause after a successful TMDB lookup that returned no match.\n\nThis prevents endless re-tries for titles that currently do not exist in TMDB.\n\nExamples: `1d`, `3d`, `7d`.", "ENABLED": "Enables metadata lookup from TMDB.\n\nTurn this off if you do not want TMDB-based title matching.", "LANGUAGE": "Preferred language for TMDB titles and metadata.\n\nExample: `en-US`.", - "RATE_LIMIT_MS": "Minimum wait time between TMDB requests in milliseconds.\n\nHigher values are gentler to the TMDB API.\n\nExamples: `250`, `500`, `1000`." + "RATE_LIMIT_MS": "Minimum wait time between TMDB requests in milliseconds.\n\nHigher values are gentler to the TMDB API.\n\nExamples: `250`, `500`, `1000`.", + "MATCH_THRESHOLD": "Minimum match score for TMDB search. Value should be between 1 - 100" }, "TRAKT_LIST_CONFIG": { "CATEGORY_NAME": "The category name used in the playlist for items from this Trakt list.", @@ -894,6 +895,7 @@ "METADATA_MAX_ATTEMPTS_RESOLVE": "Max resolve attempts", "METADATA_MAX_QUEUE_SIZE": "Max queue size", "METADATA_MAX_RESOLVE_RETRY_BACKOFF": "Max resolve retry backoff", + "METADATA_NO_CHANGE_CACHE_TTL_SECS": "No-change cache TTL (seconds)", "METADATA_PATH": "Metadata Path", "METADATA_PROBE_COOLDOWN": "Probe cooldown", "METADATA_PROBE_RETRY_BACKOFF_STEP_1": "Probe backoff step 1", diff --git a/frontend/src/app/components/config/metadata_update_config_view.rs b/frontend/src/app/components/config/metadata_update_config_view.rs index c3f245149..377e0827c 100644 --- a/frontend/src/app/components/config/metadata_update_config_view.rs +++ b/frontend/src/app/components/config/metadata_update_config_view.rs @@ -12,8 +12,8 @@ use crate::{ context::ConfigContext, }, config_field, config_field_bool, config_field_optional, edit_field_bool, edit_field_number, - edit_field_number_option_u64, edit_field_number_u64, edit_field_number_u8, edit_field_number_usize, - edit_field_text, edit_field_text_option, generate_form_reducer, + edit_field_number_option_u64, edit_field_number_u16, edit_field_number_u64, edit_field_number_u8, + edit_field_number_usize, edit_field_text, edit_field_text_option, generate_form_reducer, i18n::use_translation, }; use shared::model::{ @@ -31,6 +31,7 @@ const LABEL_TMDB_COOLDOWN: &str = "LABEL.METADATA_TMDB_COOLDOWN"; const LABEL_RETRY_DELAY: &str = "LABEL.METADATA_RETRY_DELAY"; const LABEL_PROBE_RETRY_LOAD_RETRY_DELAY: &str = "LABEL.METADATA_PROBE_RETRY_LOAD_RETRY_DELAY"; const LABEL_WORKER_IDLE_TIMEOUT: &str = "LABEL.METADATA_WORKER_IDLE_TIMEOUT"; +const LABEL_NO_CHANGE_CACHE_TTL_SECS: &str = "LABEL.METADATA_NO_CHANGE_CACHE_TTL_SECS"; const LABEL_PROBE_RETRY_BACKOFF_STEP_1: &str = "LABEL.METADATA_PROBE_RETRY_BACKOFF_STEP_1"; const LABEL_PROBE_RETRY_BACKOFF_STEP_2: &str = "LABEL.METADATA_PROBE_RETRY_BACKOFF_STEP_2"; const LABEL_PROBE_RETRY_BACKOFF_STEP_3: &str = "LABEL.METADATA_PROBE_RETRY_BACKOFF_STEP_3"; @@ -55,6 +56,7 @@ const LABEL_ENABLED: &str = "LABEL.ENABLED"; const LABEL_API_KEY: &str = "LABEL.API_KEY"; const LABEL_RATE_LIMIT_MS: &str = "LABEL.RATE_LIMIT_MS"; const LABEL_CACHE_DURATION_DAYS: &str = "LABEL.CACHE_DURATION_DAYS"; +const LABEL_TMDB_MATCH_THRESHOLD: &str = "LABEL.MATCH_THRESHOLD"; const LABEL_LANGUAGE: &str = "LABEL.LANGUAGE"; const BACKOFF_JITTER_PERCENT_VALIDATION_KEY: &str = "BACKOFF_JITTER_PERCENT_VALIDATION"; @@ -66,6 +68,7 @@ generate_form_reducer!( RetryDelay => retry_delay: String, MaxQueueSize => max_queue_size: usize, WorkerIdleTimeout => worker_idle_timeout: String, + NoChangeCacheTtlSecs => no_change_cache_ttl_secs: u64, } ); @@ -126,6 +129,7 @@ generate_form_reducer!( CacheDurationDays => cache_duration_days: u32, Language => language: String, Cooldown => cooldown: String, + MatchThreshold => match_threshold: u16, } ); @@ -234,6 +238,17 @@ pub fn MetadataUpdateConfigView() -> Html { { config_field!(ffprobe, translate.t(LABEL_FFPROBE_LIVE_PROBE_SIZE), live_probe_size) } + +

{translate.t(LABEL_PROBE)}

+ { config_field!(probe, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts) } + { config_field!(probe, translate.t(LABEL_BACKOFF_JITTER_PERCENT), backoff_jitter_percent) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_1), retry_backoff_step_1) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_2), retry_backoff_step_2) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_3), retry_backoff_step_3) } + { config_field!(probe, translate.t(LABEL_PROBE_RETRY_LOAD_RETRY_DELAY), retry_load_retry_delay) } + { config_field!(probe, translate.t(LABEL_PROBE_COOLDOWN), cooldown) } + +

{translate.t(LABEL_TMDB)}

{ config_field_bool!(tmdb, translate.t(LABEL_ENABLED), enabled) } @@ -242,6 +257,8 @@ pub fn MetadataUpdateConfigView() -> Html { { config_field!(tmdb, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days) } { config_field!(tmdb, translate.t(LABEL_LANGUAGE), language) } { config_field!(tmdb, translate.t(LABEL_TMDB_COOLDOWN), cooldown) } + { config_field!(tmdb, translate.t(LABEL_TMDB_MATCH_THRESHOLD), match_threshold) } + @@ -250,6 +267,7 @@ pub fn MetadataUpdateConfigView() -> Html { { config_field!(form_state.form, translate.t(LABEL_RETRY_DELAY), retry_delay) } { config_field!(form_state.form, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size) } { config_field!(form_state.form, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout) } + { config_field!(form_state.form, translate.t(LABEL_NO_CHANGE_CACHE_TTL_SECS), no_change_cache_ttl_secs) } @@ -266,17 +284,6 @@ pub fn MetadataUpdateConfigView() -> Html { { config_field!(resolve, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), exhaustion_reset_gap) } - -

{translate.t(LABEL_PROBE)}

- { config_field!(probe, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts) } - { config_field!(probe, translate.t(LABEL_BACKOFF_JITTER_PERCENT), backoff_jitter_percent) } - { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_1), retry_backoff_step_1) } - { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_2), retry_backoff_step_2) } - { config_field!(probe, translate.t(LABEL_PROBE_RETRY_BACKOFF_STEP_3), retry_backoff_step_3) } - { config_field!(probe, translate.t(LABEL_PROBE_RETRY_LOAD_RETRY_DELAY), retry_load_retry_delay) } - { config_field!(probe, translate.t(LABEL_PROBE_COOLDOWN), cooldown) } - - } }; @@ -302,38 +309,6 @@ pub fn MetadataUpdateConfigView() -> Html { { edit_field_text!(ffprobe_state, translate.t(LABEL_FFPROBE_LIVE_PROBE_SIZE), live_probe_size, FfprobeConfigFormAction::LiveProbeSize) } - -

{translate.t(LABEL_TMDB)}

- { edit_field_bool!(tmdb_state, translate.t(LABEL_ENABLED), enabled, TmdbConfigFormAction::Enabled) } - { edit_field_text_option!(tmdb_state, translate.t(LABEL_API_KEY), api_key, TmdbConfigFormAction::ApiKey, true) } - { edit_field_number_u64!(tmdb_state, translate.t(LABEL_RATE_LIMIT_MS), rate_limit_ms, TmdbConfigFormAction::RateLimitMs) } - { edit_field_number!(tmdb_state, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days, TmdbConfigFormAction::CacheDurationDays) } - { edit_field_text!(tmdb_state, translate.t(LABEL_LANGUAGE), language, TmdbConfigFormAction::Language) } - { edit_field_text!(tmdb_state, translate.t(LABEL_TMDB_COOLDOWN), cooldown, TmdbConfigFormAction::Cooldown) } - - - -

{translate.t(LABEL_SETTINGS)}

- { edit_field_text!(form_state, translate.t(LABEL_METADATA_PATH), cache_path, MetadataUpdateConfigFormAction::Path) } - { edit_field_text!(form_state, translate.t(LABEL_RETRY_DELAY), retry_delay, MetadataUpdateConfigFormAction::RetryDelay) } - { edit_field_number_usize!(form_state, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size, MetadataUpdateConfigFormAction::MaxQueueSize) } - { edit_field_text!(form_state, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout, MetadataUpdateConfigFormAction::WorkerIdleTimeout) } - - - -

{translate.t(LABEL_LOG)}

- { edit_field_text!(log_state, translate.t(LABEL_QUEUE_LOG_INTERVAL), queue_interval, MetadataLogConfigFormAction::QueueInterval) } - { edit_field_text!(log_state, translate.t(LABEL_PROGRESS_LOG_INTERVAL), progress_interval, MetadataLogConfigFormAction::ProgressInterval) } - - - -

{translate.t(LABEL_RESOLVE)}

- { edit_field_number_u8!(resolve_state, translate.t(LABEL_MAX_ATTEMPTS_RESOLVE), max_attempts, ResolveConfigFormAction::MaxAttempts) } - { edit_field_text!(resolve_state, translate.t(LABEL_RESOLVE_MIN_RETRY_BASE), min_retry_base, ResolveConfigFormAction::MinRetryBase) } - { edit_field_text!(resolve_state, translate.t(LABEL_MAX_RESOLVE_RETRY_BACKOFF), max_retry_backoff, ResolveConfigFormAction::MaxRetryBackoff) } - { edit_field_text!(resolve_state, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), exhaustion_reset_gap, ResolveConfigFormAction::ExhaustionResetGap) } - -

{translate.t(LABEL_PROBE)}

{ edit_field_number_u8!(probe_state, translate.t(LABEL_MAX_ATTEMPTS_PROBE), max_attempts, ProbeConfigFormAction::MaxAttempts) } @@ -376,6 +351,40 @@ pub fn MetadataUpdateConfigView() -> Html { { edit_field_text!(probe_state, translate.t(LABEL_PROBE_COOLDOWN), cooldown, ProbeConfigFormAction::Cooldown) } + +

{translate.t(LABEL_TMDB)}

+ { edit_field_bool!(tmdb_state, translate.t(LABEL_ENABLED), enabled, TmdbConfigFormAction::Enabled) } + { edit_field_text_option!(tmdb_state, translate.t(LABEL_API_KEY), api_key, TmdbConfigFormAction::ApiKey, true) } + { edit_field_number_u64!(tmdb_state, translate.t(LABEL_RATE_LIMIT_MS), rate_limit_ms, TmdbConfigFormAction::RateLimitMs) } + { edit_field_number!(tmdb_state, translate.t(LABEL_CACHE_DURATION_DAYS), cache_duration_days, TmdbConfigFormAction::CacheDurationDays) } + { edit_field_text!(tmdb_state, translate.t(LABEL_LANGUAGE), language, TmdbConfigFormAction::Language) } + { edit_field_text!(tmdb_state, translate.t(LABEL_TMDB_COOLDOWN), cooldown, TmdbConfigFormAction::Cooldown) } + { edit_field_number_u16!(tmdb_state, translate.t(LABEL_TMDB_MATCH_THRESHOLD), match_threshold, TmdbConfigFormAction::MatchThreshold) } + + + +

{translate.t(LABEL_SETTINGS)}

+ { edit_field_text!(form_state, translate.t(LABEL_METADATA_PATH), cache_path, MetadataUpdateConfigFormAction::Path) } + { edit_field_text!(form_state, translate.t(LABEL_RETRY_DELAY), retry_delay, MetadataUpdateConfigFormAction::RetryDelay) } + { edit_field_number_usize!(form_state, translate.t(LABEL_MAX_QUEUE_SIZE), max_queue_size, MetadataUpdateConfigFormAction::MaxQueueSize) } + { edit_field_text!(form_state, translate.t(LABEL_WORKER_IDLE_TIMEOUT), worker_idle_timeout, MetadataUpdateConfigFormAction::WorkerIdleTimeout) } + { edit_field_number_u64!(form_state, translate.t(LABEL_NO_CHANGE_CACHE_TTL_SECS), no_change_cache_ttl_secs, MetadataUpdateConfigFormAction::NoChangeCacheTtlSecs) } + + + +

{translate.t(LABEL_LOG)}

+ { edit_field_text!(log_state, translate.t(LABEL_QUEUE_LOG_INTERVAL), queue_interval, MetadataLogConfigFormAction::QueueInterval) } + { edit_field_text!(log_state, translate.t(LABEL_PROGRESS_LOG_INTERVAL), progress_interval, MetadataLogConfigFormAction::ProgressInterval) } + + + +

{translate.t(LABEL_RESOLVE)}

+ { edit_field_number_u8!(resolve_state, translate.t(LABEL_MAX_ATTEMPTS_RESOLVE), max_attempts, ResolveConfigFormAction::MaxAttempts) } + { edit_field_text!(resolve_state, translate.t(LABEL_RESOLVE_MIN_RETRY_BASE), min_retry_base, ResolveConfigFormAction::MinRetryBase) } + { edit_field_text!(resolve_state, translate.t(LABEL_MAX_RESOLVE_RETRY_BACKOFF), max_retry_backoff, ResolveConfigFormAction::MaxRetryBackoff) } + { edit_field_text!(resolve_state, translate.t(LABEL_RESOLVE_EXHAUSTION_RESET_GAP), exhaustion_reset_gap, ResolveConfigFormAction::ExhaustionResetGap) } + + } }; diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 313b2145a..8a0f52f4d 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -33,4 +33,6 @@ paste = "1.0.15" [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.89" -getrandom = { version = "0.4.1", features = ["wasm_js"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +getrandom = "0.4.1" diff --git a/shared/src/model/config/epg_smart_match.rs b/shared/src/model/config/epg_smart_match.rs index 7dcf38d2d..73918941f 100644 --- a/shared/src/model/config/epg_smart_match.rs +++ b/shared/src/model/config/epg_smart_match.rs @@ -2,8 +2,8 @@ use crate::{ error::TuliproxError, info_err, utils::{ - default_best_match_threshold, default_match_threshold, is_blank_optional_string, - is_default_best_match_threshold, is_default_match_threshold, is_false, + default_epg_best_match_threshold, default_epg_match_threshold, is_blank_optional_string, + is_default_epg_best_match_threshold, is_default_epg_match_threshold, is_false, }, }; use log::warn; @@ -49,9 +49,9 @@ pub struct EpgSmartMatchConfigDto { pub name_prefix_separator: Option>, #[serde(default, skip_serializing_if = "is_false")] pub fuzzy_matching: bool, - #[serde(default = "default_match_threshold", skip_serializing_if = "is_default_match_threshold")] + #[serde(default = "default_epg_match_threshold", skip_serializing_if = "is_default_epg_match_threshold")] pub match_threshold: u16, - #[serde(default = "default_best_match_threshold", skip_serializing_if = "is_default_best_match_threshold")] + #[serde(default = "default_epg_best_match_threshold", skip_serializing_if = "is_default_epg_best_match_threshold")] pub best_match_threshold: u16, } impl Default for EpgSmartMatchConfigDto { @@ -63,8 +63,8 @@ impl Default for EpgSmartMatchConfigDto { name_prefix: EpgNamePrefix::default(), name_prefix_separator: None, fuzzy_matching: false, - match_threshold: default_match_threshold(), - best_match_threshold: default_best_match_threshold(), + match_threshold: default_epg_match_threshold(), + best_match_threshold: default_epg_best_match_threshold(), } } } diff --git a/shared/src/model/config/metadata_update.rs b/shared/src/model/config/metadata_update.rs index 2a14798d6..1f57980e9 100644 --- a/shared/src/model/config/metadata_update.rs +++ b/shared/src/model/config/metadata_update.rs @@ -6,26 +6,27 @@ use crate::{ default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_max_attempts_probe, default_metadata_max_attempts_resolve, default_metadata_max_queue_size, - default_metadata_max_resolve_retry_backoff, default_metadata_path, default_metadata_probe_cooldown, - default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, - default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, - default_metadata_progress_log_interval, default_metadata_queue_log_interval, - default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, - default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, - default_tmdb_api_key, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_rate_limit_ms, - deserialize_as_string, is_default_metadata_backoff_jitter_percent, - is_default_metadata_ffprobe_analyze_duration, is_default_metadata_ffprobe_live_analyze_duration, - is_default_metadata_ffprobe_live_probe_size, is_default_metadata_ffprobe_probe_size, - is_default_metadata_max_attempts_probe, is_default_metadata_max_attempts_resolve, - is_default_metadata_max_queue_size, is_default_metadata_max_resolve_retry_backoff, is_default_metadata_path, - is_default_metadata_probe_cooldown, is_default_metadata_probe_retry_backoff_step_1, + default_metadata_max_resolve_retry_backoff, default_metadata_no_change_cache_ttl_secs, default_metadata_path, + default_metadata_probe_cooldown, default_metadata_probe_retry_backoff_step_1, + default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, + default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, + default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, + default_metadata_resolve_min_retry_base, default_metadata_retry_delay, default_metadata_tmdb_cooldown, + default_metadata_worker_idle_timeout, default_tmdb_api_key, default_tmdb_cache_duration_days, + default_tmdb_language, default_tmdb_match_threshold, default_tmdb_rate_limit_ms, deserialize_as_string, + is_default_metadata_backoff_jitter_percent, is_default_metadata_ffprobe_analyze_duration, + is_default_metadata_ffprobe_live_analyze_duration, is_default_metadata_ffprobe_live_probe_size, + is_default_metadata_ffprobe_probe_size, is_default_metadata_max_attempts_probe, + is_default_metadata_max_attempts_resolve, is_default_metadata_max_queue_size, + is_default_metadata_max_resolve_retry_backoff, is_default_metadata_no_change_cache_ttl_secs, + is_default_metadata_path, is_default_metadata_probe_cooldown, is_default_metadata_probe_retry_backoff_step_1, is_default_metadata_probe_retry_backoff_step_2, is_default_metadata_probe_retry_backoff_step_3, is_default_metadata_probe_retry_load_retry_delay, is_default_metadata_progress_log_interval, is_default_metadata_queue_log_interval, is_default_metadata_resolve_exhaustion_reset_gap, is_default_metadata_resolve_min_retry_base, is_default_metadata_retry_delay, is_default_metadata_tmdb_cooldown, is_default_metadata_worker_idle_timeout, is_default_tmdb_cache_duration_days, is_default_tmdb_language, - is_default_tmdb_rate_limit_ms, is_false, is_tmdb_default_api_key, parse_duration_seconds, parse_size_base_2, - TMDB_API_KEY, + is_default_tmdb_match_threshold, is_default_tmdb_rate_limit_ms, is_false, is_tmdb_default_api_key, + parse_duration_seconds, parse_size_base_2, TMDB_API_KEY, }, }; @@ -64,6 +65,11 @@ pub struct MetadataUpdateConfigDto { pub worker_idle_timeout: String, #[serde(default = "default_metadata_max_queue_size", skip_serializing_if = "is_default_metadata_max_queue_size")] pub max_queue_size: usize, + #[serde( + default = "default_metadata_no_change_cache_ttl_secs", + skip_serializing_if = "is_default_metadata_no_change_cache_ttl_secs" + )] + pub no_change_cache_ttl_secs: u64, } impl Default for MetadataUpdateConfigDto { @@ -78,6 +84,7 @@ impl Default for MetadataUpdateConfigDto { retry_delay: default_metadata_retry_delay(), worker_idle_timeout: default_metadata_worker_idle_timeout(), max_queue_size: default_metadata_max_queue_size(), + no_change_cache_ttl_secs: default_metadata_no_change_cache_ttl_secs(), } } } @@ -425,6 +432,8 @@ pub struct TmdbConfigDto { pub language: String, #[serde(default = "default_metadata_tmdb_cooldown", skip_serializing_if = "is_default_metadata_tmdb_cooldown")] pub cooldown: String, + #[serde(default = "default_tmdb_match_threshold", skip_serializing_if = "is_default_tmdb_match_threshold")] + pub match_threshold: u16, } impl Default for TmdbConfigDto { @@ -436,6 +445,7 @@ impl Default for TmdbConfigDto { cache_duration_days: default_tmdb_cache_duration_days(), language: default_tmdb_language(), cooldown: default_metadata_tmdb_cooldown(), + match_threshold: default_tmdb_match_threshold(), } } } @@ -448,6 +458,7 @@ impl TmdbConfigDto { && self.cache_duration_days == default_tmdb_cache_duration_days() && self.language == default_tmdb_language() && self.cooldown == default_metadata_tmdb_cooldown() + && self.match_threshold == default_tmdb_match_threshold() } fn clean(&mut self) { @@ -465,6 +476,7 @@ impl TmdbConfigDto { let cooldown_secs = MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "tmdb.cooldown")?; self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); + self.match_threshold = self.match_threshold.clamp(0, 100); Ok(()) } } @@ -480,6 +492,7 @@ impl MetadataUpdateConfigDto { && self.retry_delay == default_metadata_retry_delay() && self.worker_idle_timeout == default_metadata_worker_idle_timeout() && self.max_queue_size == default_metadata_max_queue_size() + && self.no_change_cache_ttl_secs == default_metadata_no_change_cache_ttl_secs() } pub fn clean(&mut self) { @@ -508,6 +521,7 @@ impl MetadataUpdateConfigDto { self.worker_idle_timeout = Self::canonicalize_seconds(worker_idle_timeout_secs); self.max_queue_size = self.max_queue_size.max(MIN_QUEUE_SIZE); + self.no_change_cache_ttl_secs = self.no_change_cache_ttl_secs.max(MIN_DURATION_SECS); self.clean(); @@ -609,6 +623,7 @@ mod tests { cfg.resolve.max_attempts = 0; cfg.probe.max_attempts = 0; cfg.max_queue_size = 0; + cfg.no_change_cache_ttl_secs = 0; cfg.ffprobe.timeout = Some(0); cfg.ffprobe.analyze_duration = "0s".to_string(); cfg.ffprobe.probe_size = "0".to_string(); @@ -619,6 +634,7 @@ mod tests { assert_eq!(cfg.resolve.max_attempts, 1); assert_eq!(cfg.probe.max_attempts, 1); assert_eq!(cfg.max_queue_size, 1); + assert_eq!(cfg.no_change_cache_ttl_secs, 1); assert_eq!(cfg.ffprobe.timeout, Some(1)); assert_eq!(cfg.ffprobe.analyze_duration, "1s"); assert_eq!(cfg.ffprobe.probe_size, "1B"); @@ -667,4 +683,23 @@ mod tests { let err_text = result.expect_err("validation should fail").to_string(); assert!(err_text.contains("ffprobe.analyze_duration")); } + + #[test] + fn tmdb_non_default_match_threshold_is_not_empty() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.tmdb.match_threshold = 90; + cfg.prepare().expect("metadata update config should remain valid"); + + assert!(!cfg.tmdb.is_empty(), "tmdb config with non-default match threshold must not be empty"); + assert!(!cfg.is_empty(), "metadata update config with non-default tmdb match threshold must not be empty"); + } + + #[test] + fn prepare_clamps_tmdb_match_threshold() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.tmdb.match_threshold = 250; + cfg.prepare().expect("metadata update config should clamp tmdb match threshold"); + + assert_eq!(cfg.tmdb.match_threshold, 100); + } } diff --git a/shared/src/model/info_doc_utils.rs b/shared/src/model/info_doc_utils.rs index 7a7521cf9..3208183aa 100644 --- a/shared/src/model/info_doc_utils.rs +++ b/shared/src/model/info_doc_utils.rs @@ -1,4 +1,3 @@ -use crate::model::xtream_const; use serde_json::Value; pub struct InfoDocUtils {} @@ -17,20 +16,6 @@ impl InfoDocUtils { digits_only.parse::().ok() } - pub fn make_bdpath_resource_url( - resource_url: Option<&str>, - bd_path: &str, - index: usize, - field_prefix: &str, - ) -> String { - if let Some(url) = resource_url { - if bd_path.starts_with("http") { - return format!("{url}/{field_prefix}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH); - } - } - bd_path.to_string() - } - pub fn limited(n: f64) -> String { if n < 0.01 { "0".to_string() @@ -48,13 +33,4 @@ impl InfoDocUtils { } Value::Array(Vec::new()) } - - pub fn make_resource_url(resource_url: Option<&str>, value: &str, field: &str) -> String { - if let Some(url) = resource_url { - if value.starts_with("http") { - return format!("{url}/{field}"); - } - } - value.to_string() - } } diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index 1bdb85db0..b4259965e 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -5,8 +5,8 @@ use crate::{ SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, XtreamInfoDocument, }, utils::{ - arc_str_option_serde, arc_str_serde, extract_extension_from_url, generate_playlist_uuid, get_provider_id, - Internable, + arc_str_option_serde, arc_str_serde, concat_path, extract_extension_from_url, generate_playlist_uuid, + get_provider_id, obfuscate_text, Internable, }, }; use enum_iterator::Sequence; @@ -633,35 +633,86 @@ pub struct XtreamMappingOptions { pub reverse_item_types: PlaylistItemTypeSet, pub username: String, pub password: String, - pub base_url: Option, + pub base_url: String, + pub web_ui_request: bool, + pub encrypt_secret: [u8; 16], } impl XtreamMappingOptions { #[inline] pub fn is_reverse(&self, item_type: PlaylistItemType) -> bool { self.reverse_item_types.is_set(item_type) } - pub fn get_resource_url( + fn build_reverse_proxy_base_url( &self, xtream_cluster: XtreamCluster, item_type: PlaylistItemType, virtual_id: VirtualId, ) -> Option { let is_reverse = self.is_reverse(item_type); - let resource_url = - if is_reverse && self.flags.contains(XtreamMappingFlags::RewriteResourceUrl) && self.base_url.is_some() { - let resource_url = format!( - "{}/resource/{}/{}/{}/{}", - self.base_url.as_ref().map_or_else(String::new, |b| b.clone()), - xtream_cluster.as_stream_type(), - self.username, - self.password, - virtual_id - ); - Some(resource_url) - } else { - None - }; - resource_url + if is_reverse && self.flags.contains(XtreamMappingFlags::RewriteResourceUrl) { + Some(format!( + "{}/resource/{}/{}/{}/{}", + self.base_url, + xtream_cluster.as_stream_type(), + self.username, + self.password, + virtual_id + )) + } else { + None + } + } + + pub fn get_resource_url( + &self, + xtream_cluster: XtreamCluster, + item_type: PlaylistItemType, + virtual_id: VirtualId, + resource_url: &str, + resource_field: &str, + ) -> String { + if self.web_ui_request { + if resource_url.is_empty() { + return resource_url.to_string(); + } + let rewrite_url = concat_path(&self.base_url, &obfuscate_text(&self.encrypt_secret, resource_url)); + return rewrite_url; + } + + let rewrite_url = self.build_reverse_proxy_base_url(xtream_cluster, item_type, virtual_id); + + if let Some(url) = rewrite_url { + if resource_url.starts_with("http://") || resource_url.starts_with("https://") { + return format!("{url}/{resource_field}"); + } + } + resource_url.to_string() + } + pub fn get_bd_path_resource_url( + &self, + xtream_cluster: XtreamCluster, + item_type: PlaylistItemType, + virtual_id: VirtualId, + resource_url: &str, + resource_field: &str, + index: usize, + ) -> String { + if self.web_ui_request { + if resource_url.is_empty() { + return resource_url.to_string(); + } + let rewrite_url = concat_path(&self.base_url, &obfuscate_text(&self.encrypt_secret, resource_url)); + return rewrite_url; + } + + let rewrite_url = self.build_reverse_proxy_base_url(xtream_cluster, item_type, virtual_id); + + if let Some(url) = rewrite_url { + if resource_url.starts_with("http://") || resource_url.starts_with("https://") { + return format!("{url}/{resource_field}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH); + } + } + resource_url.to_string() } } diff --git a/shared/src/model/playlist_document.rs b/shared/src/model/playlist_document.rs index 9bf5adb35..d34a22679 100644 --- a/shared/src/model/playlist_document.rs +++ b/shared/src/model/playlist_document.rs @@ -119,80 +119,84 @@ impl XtreamPlaylistItem { return doc.to_info_document(options, self.item_type, self.virtual_id, self.category_id); } } - let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); - self.to_info_document_no_props(resource_url) + self.to_info_document_no_props(options) } - fn to_info_document_no_props(&self, resource_url: Option) -> XtreamInfoDocument { - let stream_icon = self.get_stream_icon(resource_url); + fn to_info_document_no_props(&self, options: &XtreamMappingOptions) -> XtreamInfoDocument { let empty_str = "".intern(); match self.xtream_cluster { XtreamCluster::Live => XtreamInfoDocument::Empty(XtreamEmptyDoc {}), - XtreamCluster::Video => XtreamInfoDocument::Video(XtreamVideoInfoDoc { - info: XtreamVideoInfoData { - kinopoisk_url: Arc::clone(&empty_str), - tmdb_id: Arc::clone(&empty_str), - name: Arc::clone(&self.title), - o_name: Arc::clone(&self.name), - cover_big: Arc::clone(&stream_icon), - movie_image: Arc::clone(&stream_icon), - release_date: Arc::clone(&empty_str), - episode_run_time: 0, - youtube_trailer: Arc::clone(&empty_str), - director: Arc::clone(&empty_str), - actors: Arc::clone(&empty_str), - cast: Arc::clone(&empty_str), - description: Arc::clone(&empty_str), - plot: Arc::clone(&empty_str), - age: Arc::clone(&empty_str), - mpaa_rating: Arc::clone(&empty_str), - rating_count_kinopoisk: 0, - country: Arc::clone(&empty_str), - genre: Arc::clone(&empty_str), - backdrop_path: vec![Arc::clone(&stream_icon)], - duration_secs: "0".intern(), - duration: Arc::clone(&empty_str), - video: Value::Array(Vec::new()), - audio: Value::Array(Vec::new()), - bitrate: 0, - rating: Arc::clone(&empty_str), - runtime: Arc::clone(&empty_str), - status: "Released".intern(), - }, - movie_data: XtreamVideoMovieData { - stream_id: self.virtual_id, - name: Arc::clone(&self.name), - added: Arc::clone(&empty_str), - category_id: self.category_id.intern(), - category_ids: vec![self.category_id], - container_extension: Arc::clone(&empty_str), - custom_sid: None, - direct_source: Arc::clone(&empty_str), - }, - }), - XtreamCluster::Series => XtreamInfoDocument::Series(XtreamSeriesInfoDoc { - seasons: Vec::new(), - info: XtreamSeriesInfoData { - name: Arc::clone(&self.title), - cover: Arc::clone(&stream_icon), - plot: Arc::clone(&empty_str), - cast: Arc::clone(&empty_str), - director: Arc::clone(&empty_str), - genre: Arc::clone(&empty_str), - release_date_alternate: Arc::clone(&empty_str), - release_date: Arc::clone(&empty_str), - last_modified: Arc::clone(&empty_str), - rating: Arc::clone(&empty_str), - rating_5based: Arc::clone(&empty_str), - backdrop_path: if stream_icon.is_empty() { vec![] } else { vec![Arc::clone(&stream_icon)] }, - tmdb: Arc::clone(&empty_str), - youtube_trailer: Arc::clone(&empty_str), - episode_run_time: empty_str, - category_id: self.category_id.intern(), - category_ids: vec![self.category_id], - }, - episodes: IndexMap::new(), - }), + XtreamCluster::Video => { + let stream_icon = self.get_stream_icon(options); + XtreamInfoDocument::Video(XtreamVideoInfoDoc { + info: XtreamVideoInfoData { + kinopoisk_url: Arc::clone(&empty_str), + tmdb_id: Arc::clone(&empty_str), + name: Arc::clone(&self.title), + o_name: Arc::clone(&self.name), + cover_big: Arc::clone(&stream_icon), + movie_image: Arc::clone(&stream_icon), + release_date: Arc::clone(&empty_str), + episode_run_time: 0, + youtube_trailer: Arc::clone(&empty_str), + director: Arc::clone(&empty_str), + actors: Arc::clone(&empty_str), + cast: Arc::clone(&empty_str), + description: Arc::clone(&empty_str), + plot: Arc::clone(&empty_str), + age: Arc::clone(&empty_str), + mpaa_rating: Arc::clone(&empty_str), + rating_count_kinopoisk: 0, + country: Arc::clone(&empty_str), + genre: Arc::clone(&empty_str), + backdrop_path: vec![Arc::clone(&stream_icon)], + duration_secs: "0".intern(), + duration: Arc::clone(&empty_str), + video: Value::Array(Vec::new()), + audio: Value::Array(Vec::new()), + bitrate: 0, + rating: Arc::clone(&empty_str), + runtime: Arc::clone(&empty_str), + status: "Released".intern(), + }, + movie_data: XtreamVideoMovieData { + stream_id: self.virtual_id, + name: Arc::clone(&self.name), + added: Arc::clone(&empty_str), + category_id: self.category_id.intern(), + category_ids: vec![self.category_id], + container_extension: Arc::clone(&empty_str), + custom_sid: None, + direct_source: Arc::clone(&empty_str), + }, + }) + } + XtreamCluster::Series => { + let stream_icon = self.get_stream_icon(options); + XtreamInfoDocument::Series(XtreamSeriesInfoDoc { + seasons: Vec::new(), + info: XtreamSeriesInfoData { + name: Arc::clone(&self.title), + cover: Arc::clone(&stream_icon), + plot: Arc::clone(&empty_str), + cast: Arc::clone(&empty_str), + director: Arc::clone(&empty_str), + genre: Arc::clone(&empty_str), + release_date_alternate: Arc::clone(&empty_str), + release_date: Arc::clone(&empty_str), + last_modified: Arc::clone(&empty_str), + rating: Arc::clone(&empty_str), + rating_5based: Arc::clone(&empty_str), + backdrop_path: if stream_icon.is_empty() { vec![] } else { vec![Arc::clone(&stream_icon)] }, + tmdb: Arc::clone(&empty_str), + youtube_trailer: Arc::clone(&empty_str), + episode_run_time: empty_str, + category_id: self.category_id.intern(), + category_ids: vec![self.category_id], + }, + episodes: IndexMap::new(), + }) + } } } @@ -205,19 +209,17 @@ impl XtreamPlaylistItem { StreamProperties::Episode(_episode) => XtreamDocument::Episode(XtreamEmptyDoc::default()), } } else { - let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); - self.to_document_no_props(resource_url) + self.to_document_no_props(options) } } fn series_to_document(&self, options: &XtreamMappingOptions, series: &SeriesStreamProperties) -> XtreamDocument { - let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); let empty_str = "".intern(); XtreamDocument::Series(XtreamSeriesDoc { num: self.channel_no, name: self.title.clone(), series_id: self.virtual_id, - cover: InfoDocUtils::make_resource_url(resource_url.as_deref(), &series.cover, "cover").intern(), + cover: self.get_stream_resource(options, &series.cover, "cover"), plot: series.plot.clone().unwrap_or_else(|| Arc::clone(&empty_str)), cast: series.cast.clone(), director: series.director.clone(), @@ -229,18 +231,27 @@ impl XtreamPlaylistItem { rating_5based: InfoDocUtils::limited(series.rating_5based).intern(), backdrop_path: series.backdrop_path.as_ref().map_or_else( || { - let res_url = InfoDocUtils::make_resource_url(resource_url.as_deref(), &series.cover, "cover"); + let res_url = self.get_stream_resource(options, &series.cover, "cover"); if res_url.is_empty() { vec![] } else { - vec![res_url.intern()] + vec![res_url] } }, |b| { b.iter() .enumerate() .map(|(idx, p)| { - InfoDocUtils::make_bdpath_resource_url(resource_url.as_deref(), p, idx, "").intern() + options + .get_bd_path_resource_url( + XtreamCluster::Series, + self.item_type, + self.virtual_id, + p, + "", + idx, + ) + .intern() }) .collect() }, @@ -254,8 +265,7 @@ impl XtreamPlaylistItem { } fn video_to_document(&self, options: &XtreamMappingOptions, video: &VideoStreamProperties) -> XtreamDocument { - let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); - let stream_icon = self.get_stream_icon(resource_url); + let stream_icon = self.get_stream_icon(options); let empty_str = "".intern(); XtreamDocument::Video(XtreamVideoDoc { num: self.channel_no, @@ -282,8 +292,7 @@ impl XtreamPlaylistItem { } fn live_to_document(&self, options: &XtreamMappingOptions, live: &LiveStreamProperties) -> XtreamDocument { - let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); - let stream_icon = self.get_stream_icon(resource_url); + let stream_icon = self.get_stream_icon(options); let empty_str = "".intern(); XtreamDocument::Live(XtreamLiveDoc { num: self.channel_no, @@ -307,10 +316,10 @@ impl XtreamPlaylistItem { }) } - fn to_document_no_props(&self, resource_url: Option) -> XtreamDocument { + fn to_document_no_props(&self, options: &XtreamMappingOptions) -> XtreamDocument { let empty_str = "".intern(); let zero_str = "0".intern(); - let stream_icon = self.get_stream_icon(resource_url); + let stream_icon = self.get_stream_icon(options); match self.xtream_cluster { XtreamCluster::Live => XtreamDocument::Live(XtreamLiveDoc { num: self.channel_no, @@ -370,15 +379,26 @@ impl XtreamPlaylistItem { } } - fn get_stream_icon(&self, resource_url: Option) -> Arc { + fn get_stream_icon(&self, options: &XtreamMappingOptions) -> Arc { if !self.logo.is_empty() { - InfoDocUtils::make_resource_url(resource_url.as_deref(), &self.logo, "logo").intern() + self.get_stream_resource(options, &self.logo, "logo") } else if !self.logo_small.is_empty() { - InfoDocUtils::make_resource_url(resource_url.as_deref(), &self.logo_small, "logo_small").intern() + self.get_stream_resource(options, &self.logo_small, "logo_small") } else { "".intern() } } + + fn get_stream_resource( + &self, + options: &XtreamMappingOptions, + resource_url: &str, + resource_field: &str, + ) -> Arc { + options + .get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id, resource_url, resource_field) + .intern() + } } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] diff --git a/shared/src/model/playlist_info_document.rs b/shared/src/model/playlist_info_document.rs index 8e0f0e1cb..fbbeca435 100644 --- a/shared/src/model/playlist_info_document.rs +++ b/shared/src/model/playlist_info_document.rs @@ -272,16 +272,16 @@ impl StreamProperties { virtual_id: VirtualId, category_id: u32, ) -> XtreamSeriesInfoDoc { - let resource_url = options.get_resource_url(XtreamCluster::Series, item_type, virtual_id); XtreamSeriesInfoDoc { seasons: if let Some(seasons) = series.details.as_ref().and_then(|d| d.seasons.as_ref()) { - self.series_seasons_to_info_document(resource_url.as_deref(), seasons) + self.series_seasons_to_info_document(options, item_type, virtual_id, seasons) } else { Vec::new() }, info: XtreamSeriesInfoData { name: Arc::clone(&series.name), - cover: InfoDocUtils::make_resource_url(resource_url.as_deref(), series.cover.as_ref(), "cover") + cover: options + .get_resource_url(XtreamCluster::Series, item_type, virtual_id, series.cover.as_ref(), "cover") .intern(), plot: series.plot.as_ref().map(Arc::clone).unwrap_or_else(|| "".intern()), cast: Arc::clone(&series.cast), @@ -296,7 +296,9 @@ impl StreamProperties { b.iter() .enumerate() .map(|(idx, p)| { - InfoDocUtils::make_bdpath_resource_url(resource_url.as_deref(), p, idx, "").intern() + options + .get_bd_path_resource_url(XtreamCluster::Series, item_type, virtual_id, p, "", idx) + .intern() }) .collect() }), @@ -307,7 +309,7 @@ impl StreamProperties { category_ids: vec![category_id], }, episodes: if let Some(episodes) = series.details.as_ref().and_then(|d| d.episodes.as_ref()) { - self.series_episodes_to_info_document(options, resource_url.as_deref(), episodes) + self.series_episodes_to_info_document(options, item_type, virtual_id, episodes) } else { IndexMap::new() }, @@ -322,9 +324,9 @@ impl StreamProperties { virtual_id: VirtualId, category_id: u32, ) -> XtreamVideoInfoDoc { - let resource_url = options.get_resource_url(XtreamCluster::Video, item_type, virtual_id); - let stream_icon = - InfoDocUtils::make_resource_url(resource_url.as_deref(), &self.get_stream_icon(), "logo").intern(); + let stream_icon = options + .get_resource_url(XtreamCluster::Video, item_type, virtual_id, &self.get_stream_icon(), "logo") + .intern(); let empty_str = "".intern(); let zero_str = "0".intern(); @@ -334,18 +336,24 @@ impl StreamProperties { tmdb_id: video.tmdb.unwrap_or_default().to_string().intern(), name: Arc::clone(&video.name), o_name: details.o_name.as_ref().map(Arc::clone).unwrap_or_else(|| Arc::clone(&empty_str)), - cover_big: InfoDocUtils::make_resource_url( - resource_url.as_deref(), - details.cover_big.as_ref().map(Arc::as_ref).unwrap_or(""), - "nfo_cover_big", - ) - .intern(), - movie_image: InfoDocUtils::make_resource_url( - resource_url.as_deref(), - details.cover_big.as_ref().map(Arc::as_ref).unwrap_or(""), - "nfo_movie_image", - ) - .intern(), + cover_big: options + .get_resource_url( + XtreamCluster::Video, + item_type, + virtual_id, + details.cover_big.as_ref().map(Arc::as_ref).unwrap_or(""), + "nfo_cover_big", + ) + .intern(), + movie_image: options + .get_resource_url( + XtreamCluster::Video, + item_type, + virtual_id, + details.movie_image.as_ref().map(Arc::as_ref).unwrap_or(""), + "nfo_movie_image", + ) + .intern(), release_date: details.release_date.as_ref().map(Arc::clone).unwrap_or_else(|| Arc::clone(&empty_str)), episode_run_time: details.episode_run_time.unwrap_or_default(), youtube_trailer: details @@ -367,7 +375,9 @@ impl StreamProperties { b.iter() .enumerate() .map(|(idx, p)| { - InfoDocUtils::make_bdpath_resource_url(resource_url.as_deref(), p, idx, "nfo_").intern() + options + .get_bd_path_resource_url(XtreamCluster::Video, item_type, virtual_id, p, "nfo_", idx) + .intern() }) .collect() }), @@ -434,7 +444,9 @@ impl StreamProperties { fn series_seasons_to_info_document( &self, - resource_url: Option<&str>, + options: &XtreamMappingOptions, + item_type: PlaylistItemType, + virtual_id: VirtualId, seasons: &[SeriesStreamDetailSeasonProperties], ) -> Vec { seasons @@ -444,37 +456,53 @@ impl StreamProperties { season_number: season.season_number, episode_count: season.episode_count.intern(), overview: season.overview.as_ref().map(|v| { - if v.starts_with("http") { - InfoDocUtils::make_resource_url( - resource_url, - v, - &build_season_field(season.season_number, "overview"), - ) - .intern() + if v.starts_with("http://") || v.starts_with("https://") { + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "overview"), + ) + .intern() } else { Arc::clone(v) } }), air_date: season.air_date.as_ref().map(Arc::clone), cover: season.cover.as_ref().map(|v| { - InfoDocUtils::make_resource_url(resource_url, v, &build_season_field(season.season_number, "cover")) + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "cover"), + ) .intern() }), cover_tmdb: season.cover_tmdb.as_ref().map(|v| { - InfoDocUtils::make_resource_url( - resource_url, - v, - &build_season_field(season.season_number, "cover_tmdb"), - ) - .intern() + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "cover_tmdb"), + ) + .intern() }), cover_big: season.cover_big.as_ref().map(|v| { - InfoDocUtils::make_resource_url( - resource_url, - v, - &build_season_field(season.season_number, "cover_big"), - ) - .intern() + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "cover_big"), + ) + .intern() }), release_date: season.air_date.as_ref().map(Arc::clone), duration: season.duration.as_ref().map(Arc::clone), @@ -485,7 +513,8 @@ impl StreamProperties { fn series_episodes_to_info_document( &self, options: &XtreamMappingOptions, - resource_url: Option<&str>, + item_type: PlaylistItemType, + virtual_id: VirtualId, episodes: &[SeriesStreamDetailEpisodeProperties], ) -> IndexMap> { let empty_str = "".intern(); @@ -501,12 +530,15 @@ impl StreamProperties { air_date: Arc::clone(&ep.release_date), crew: ep.crew.as_ref().map(Arc::clone), rating: ep.rating.unwrap_or_default(), - movie_image: InfoDocUtils::make_resource_url( - resource_url, - &ep.movie_image, - &build_season_episode_field(ep.season, ep.episode_num, "movie_image"), - ) - .intern(), + movie_image: options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + &ep.movie_image, + &build_season_episode_field(ep.season, ep.episode_num, "movie_image"), + ) + .intern(), duration: Arc::clone(&ep.duration), duration_secs: ep.duration_secs, video: InfoDocUtils::build_value(ep.video.as_ref().map(Arc::as_ref)), diff --git a/shared/src/utils/bitset.rs b/shared/src/utils/bitset.rs index b1eb0a03d..608d5eb7a 100644 --- a/shared/src/utils/bitset.rs +++ b/shared/src/utils/bitset.rs @@ -71,6 +71,12 @@ macro_rules! create_bitset { self.0 == 0 } + /// Returns true if every bit set in `self` is also set in `other`. + #[inline(always)] + pub fn is_subset_of(&self, other: &Self) -> bool { + (self.0 & other.0) == self.0 + } + #[inline] pub fn clear(&mut self) { self.0 = 0; diff --git a/shared/src/utils/crypto.rs b/shared/src/utils/crypto.rs new file mode 100644 index 000000000..90bb971f0 --- /dev/null +++ b/shared/src/utils/crypto.rs @@ -0,0 +1,43 @@ +use base64::{engine::general_purpose, Engine as _}; + +pub fn encode_base64_string(input: &[u8]) -> String { general_purpose::URL_SAFE_NO_PAD.encode(input) } + +pub fn decode_base64_string(input: &str) -> Vec { + general_purpose::URL_SAFE_NO_PAD.decode(input).unwrap_or_else(|_| input.as_bytes().to_vec()) +} + +pub fn xor_bytes(secret: &[u8], data: &[u8]) -> Vec { + if secret.is_empty() { + return data.to_vec(); + } + data.iter().enumerate().map(|(i, &b)| b ^ secret[i % secret.len()]).collect() +} + +pub fn obfuscate_text(secret: &[u8], text: &str) -> String { encode_base64_string(&xor_bytes(secret, text.as_bytes())) } + +pub fn deobfuscate_text(secret: &[u8], text: &str) -> Result { + let data = xor_bytes(secret, &decode_base64_string(text)); + if let Ok(result) = String::from_utf8(data) { + Ok(result) + } else { + Err(text.to_string()) + } +} + +#[cfg(test)] +mod tests { + use crate::utils::{deobfuscate_text, obfuscate_text}; + + #[test] + fn test_obfuscate() { + let mut secret = [0u8; 16]; + for x in &mut secret { + *x = fastrand::u8(..); + } + let plain = "hello world"; + let encrypted = obfuscate_text(&secret, plain); + let decrypted = deobfuscate_text(&secret, &encrypted).unwrap(); + + assert_eq!(decrypted, plain); + } +} diff --git a/shared/src/utils/default_utils.rs b/shared/src/utils/default_utils.rs index 94bdd13dc..f029e31e6 100644 --- a/shared/src/utils/default_utils.rs +++ b/shared/src/utils/default_utils.rs @@ -51,24 +51,47 @@ pub const fn is_default_resource_retry_backoff_multiplier(v: &f64) -> bool { (*v - default_resource_retry_backoff_multiplier()).abs() < F64_DEFAULT_EPSILON } -pub fn default_secret() -> String { - let mut out = [0u8; 16]; - for x in &mut out { - *x = fastrand::u8(..); +fn fill_with_secure_random_bytes(out: &mut [u8]) { + #[cfg(target_arch = "wasm32")] + { + for byte in out { + *byte = fastrand::u8(..); + } + } + + #[cfg(not(target_arch = "wasm32"))] + if let Err(err) = getrandom::fill(out) { + panic!("failed to generate secure random bytes: {err}"); } - out.iter().map(|b| format!("{:02X}", b)).collect() } +pub fn generate_default_access_secret() -> [u8; 32] { + let mut out = [0u8; 32]; + fill_with_secure_random_bytes(&mut out); + out +} + +pub fn generate_default_encrypt_secret() -> [u8; 16] { + let mut out = [0u8; 16]; + fill_with_secure_random_bytes(&mut out); + out +} + +pub fn default_secret() -> String { generate_default_encrypt_secret().iter().map(|b| format!("{:02X}", b)).collect() } + pub const fn default_kick_secs() -> u64 { 90 } pub const fn is_default_kick_secs(v: &u64) -> bool { *v == default_kick_secs() } /// 30 minutes by default; `0` still means “no expiration.” pub const fn default_token_ttl_mins() -> u32 { 30 } pub const fn is_default_token_ttl_mins(v: &u32) -> bool { *v == default_token_ttl_mins() } -pub const fn default_match_threshold() -> u16 { 80 } -pub const fn is_default_match_threshold(v: &u16) -> bool { *v == default_match_threshold() } -pub const fn default_best_match_threshold() -> u16 { 95 } -pub const fn is_default_best_match_threshold(v: &u16) -> bool { *v == default_best_match_threshold() } +pub const fn default_epg_match_threshold() -> u16 { 80 } +pub const fn is_default_epg_match_threshold(v: &u16) -> bool { *v == default_epg_match_threshold() } +pub const fn default_epg_best_match_threshold() -> u16 { 95 } +pub const fn is_default_epg_best_match_threshold(v: &u16) -> bool { *v == default_epg_best_match_threshold() } + +pub const fn default_tmdb_match_threshold() -> u16 { 86 } +pub const fn is_default_tmdb_match_threshold(v: &u16) -> bool { *v == default_tmdb_match_threshold() } pub const TMDB_API_KEY: &str = "4219e299c89411838049ab0dab19ebd5"; pub fn default_tmdb_api_key() -> Option { Some(TMDB_API_KEY.to_string()) } @@ -176,6 +199,10 @@ pub const fn is_default_metadata_backoff_jitter_percent(v: &u8) -> bool { } pub const fn default_metadata_max_queue_size() -> usize { 100_000 } pub const fn is_default_metadata_max_queue_size(v: &usize) -> bool { *v == default_metadata_max_queue_size() } +pub const fn default_metadata_no_change_cache_ttl_secs() -> u64 { 3600 } +pub const fn is_default_metadata_no_change_cache_ttl_secs(v: &u64) -> bool { + *v == default_metadata_no_change_cache_ttl_secs() +} pub fn default_metadata_ffprobe_analyze_duration() -> String { "10s".to_string() } pub fn is_default_metadata_ffprobe_analyze_duration(v: &String) -> bool { *v == default_metadata_ffprobe_analyze_duration() diff --git a/shared/src/utils/mod.rs b/shared/src/utils/mod.rs index f05cbf29d..bd83d29e2 100644 --- a/shared/src/utils/mod.rs +++ b/shared/src/utils/mod.rs @@ -1,5 +1,6 @@ mod bitset; mod constants; +mod crypto; mod default_utils; mod directed_graph; mod hash_utils; @@ -15,8 +16,9 @@ mod string_utils; mod time_utils; pub use self::{ - constants::*, default_utils::*, directed_graph::*, hash_utils::*, hdhomerun_utils::*, json_utils::*, net_utils::*, - number_utils::*, request::*, serde_utils::*, size_utils::*, string_interner::*, string_utils::*, time_utils::*, + constants::*, crypto::*, default_utils::*, directed_graph::*, hash_utils::*, hdhomerun_utils::*, json_utils::*, + net_utils::*, number_utils::*, request::*, serde_utils::*, size_utils::*, string_interner::*, string_utils::*, + time_utils::*, }; use std::fmt::Display;