From d0ce95eb3c4a60d47f1f7b4401c0364d30e4de8e Mon Sep 17 00:00:00 2001 From: euzu <33094714+euzu@users.noreply.github.com> Date: Mon, 23 Feb 2026 17:00:11 +0100 Subject: [PATCH] Fix/web UI save overrides templates (#604) - WebUI save overrides templates - Deadlock during playlist update and metadata background task - Global templates like mappings --- CHANGELOG.md | 8 + README.md | 54 +- backend/src/api/config_file.rs | 101 ++- backend/src/api/config_watch.rs | 17 +- backend/src/api/endpoints/v1_api_config.rs | 55 +- backend/src/api/main_api.rs | 216 +++---- .../src/api/model/active_provider_manager.rs | 5 +- backend/src/api/model/app_state.rs | 4 + .../src/api/model/metadata_update_manager.rs | 592 +++++++++++------- backend/src/api/setup_api.rs | 123 +++- backend/src/main.rs | 13 + backend/src/messaging.rs | 2 + backend/src/model/config/app.rs | 18 + backend/src/model/config/base.rs | 2 + backend/src/model/config/library.rs | 106 ++-- backend/src/model/config/rename.rs | 67 +- backend/src/model/config/target.rs | 84 ++- backend/src/processing/playlist_watch.rs | 40 +- backend/src/processing/processor/playlist.rs | 58 +- backend/src/processing/processor/sort.rs | 242 +++---- .../src/processing/processor/xtream_series.rs | 198 +++--- .../src/processing/processor/xtream_vod.rs | 204 +++--- backend/src/repository/user_repository.rs | 2 + backend/src/utils/file/config_reader.rs | 486 ++++++++++++-- backend/src/utils/file/file_utils.rs | 36 +- backend/src/utils/file/mapping_reader.rs | 51 +- backend/src/utils/file/mod.rs | 4 +- backend/src/utils/file/template_reader.rs | 114 ++++ backend/src/utils/telegram.rs | 207 +++++- frontend/public/assets/i18n/en.json | 2 + .../app/components/config/config_update.rs | 70 ++- .../src/app/components/config/config_view.rs | 14 +- .../app/components/config/main_config_view.rs | 4 + .../app/components/filter/filter_editor.rs | 8 +- .../src/app/components/filter/filter_input.rs | 11 +- .../src/app/components/setup/setup_helpers.rs | 6 +- frontend/src/services/config_service.rs | 43 +- shared/src/foundation/filter.rs | 28 +- shared/src/foundation/mapper.rs | 8 +- shared/src/model/config/app_config.rs | 3 +- shared/src/model/config/base.rs | 44 +- shared/src/model/config/config_type.rs | 2 + shared/src/model/config/favourites.rs | 2 +- shared/src/model/config/mod.rs | 2 + shared/src/model/config/paths.rs | 4 + shared/src/model/config/rename.rs | 73 ++- shared/src/model/config/sort.rs | 4 +- shared/src/model/config/source.rs | 100 ++- shared/src/model/config/target.rs | 10 +- .../src/model/config/template_definition.rs | 8 + shared/src/model/mapping.rs | 84 ++- shared/src/utils/constants.rs | 1 + 52 files changed, 2513 insertions(+), 1127 deletions(-) create mode 100644 backend/src/utils/file/template_reader.rs create mode 100644 shared/src/model/config/template_definition.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 23bc1b032..0d8f38566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,11 +96,19 @@ active URL of the specified provider. - Added `epg_request_timeshift: [-+]hh:mm or TimeZone`, example `Europe/Paris`, `America/New_York`, `-2:30`(-2h30m), `+0:15` (15m), `2` (2h), `:30` (30m), `:3` (3m) - **Extended scheduler** to support `Local Library` scans. Scheduler can now trigger automatic library scans alongside playlist updates. +- **Centralized Pattern Templates**: Added a global template collection that is loaded from `config.yml -> template_path` (file or directory) and shared + across sources and mappings. +- **Template Backward Compatibility**: Existing inline templates in `source.yml` and `mapping.yml` are still loaded and merged during read/validation. +- **Template-Aware Hot Reload**: File watcher now tracks template files/directories and reapplies sources/mappings when templates change. +- **Setup Validation Improvement**: Setup mode validates source configuration against the global template collection and persists template definitions + separately. +- Added `-T, --template` to override `template_path` on startup. ## ⚙️ New Settings - **config.yml**: - Added `video.ffprobe_enabled` (default: false) and `video.ffprobe_timeout`. + - Added `template_path` (optional): path to a template file (`template.yml`) or directory (`template.d` style). - **source.yml (input options)**: - Added `resolve_tmdb`: Triggers TMDB lookup if ID is missing. - Added `probe_stream`: Triggers ffprobe if technical info is missing. diff --git a/README.md b/README.md index 567920038..1282bb82d 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Options: -c, --config The config file -i, --source The source config file -m, --mapping The mapping file + -T, --template The template file or template directory -t, --target The target to process -a, --api-proxy The user file -s, --server Run in server mode @@ -77,7 +78,7 @@ Options: --dbx Dump database with content type: xtream --dbm Dump database with content type: m3u --dbe Dump database with content type: epg - --dbv Dump database with content type: taregt id mapping + --dbv Dump database with content type: target id mapping ``` @@ -101,6 +102,8 @@ Top level entries in the config files are: - `video` _optional_ - `schedules` _optional_ - `backup_dir` _optional_ +- `mapping_path` _optional_, file or directory path for mappings (`mapping.yml` by default) +- `template_path` _optional_, file or directory path for global templates (`template.yml` by default) - `update_on_boot` _optional_ - `web_ui` _optional_ - `reverse_proxy` _optional_ @@ -136,6 +139,34 @@ With this configuration, you should create a `data` directory where you execute Be aware that different configurations (e.g. user bouquets) along the playlists are stored in this directory. +### 1.3.1 `mapping_path` and `template_path` + +Tuliprox supports centralized loading of mappings and templates via `config.yml`: + +- `mapping_path`: single file or directory +- `template_path`: single file or directory + +Defaults: + +- mappings: `mapping.yml` +- templates: `template.yml` + +If a path points to a directory, all `*.yml` files are loaded in **alphanumeric** order and merged. +Template names must be unique globally. + +Example `template.yml`: + +```yaml +templates: + - name: ALL_CHANNELS + value: Group ~ ".*" +``` + +CLI overrides: + +- `-m, --mapping` overrides `mapping_path` +- `-T, --template` overrides `template_path` + ## 1.4 Provider Failover & Rotation Tuliprox supports robust failover mechanisms for streaming providers. If a provider has multiple URLs defined (or aliases), Tuliprox can automatically @@ -1018,15 +1049,18 @@ sources: Has the following top level entries: -- `templates` _optional_ +- `templates` _optional_ (legacy/backward-compatible; use global `template_path` for new setups) - `inputs` - `sources` ### 2.1 `templates` -If you have a lot of repeats in you regexps, you can use `templates` to make your regexps cleaner. +If you have a lot of repeats in your regexps, you can use templates to make your regexps cleaner. You can reference other templates in templates with `!name!`. +For new configurations, prefer a centralized `template.yml` / `template.d` via `config.yml.template_path`. +Inline `source.yml.templates` is still loaded for backward compatibility. + ```yaml templates: - {name: delimiter, value: '[\s_-]*' } @@ -1811,7 +1845,10 @@ The files are loaded in **alphanumeric** order. The filename or path can be given as `-m` argument. (See Mappings section) -Default mapping file is `maping.yml` +Global templates can be loaded via `template_path` in `config.yml` (or CLI `-T`). +Inline templates in `source.yml` and `mapping.yml` are still accepted for backward compatibility. + +Default mapping file is `mapping.yml` ## Example source.yml file @@ -1930,7 +1967,7 @@ messaging: Has the root item `mappings` which has the following top level entries: -- `templates` _optional_ +- `templates` _optional_ (legacy/backward-compatible; use global `template_path` for new setups) - `mapping` _mandatory_ Instead of using a single `mapping.yml` file, you can use multiple mapping files @@ -1942,13 +1979,16 @@ The files are loaded in **alphanumeric** order. The filename or path can be given as `-m` argument. (See Mappings section) -Default mapping file is `maping.yml` +Default mapping file is `mapping.yml` ### 3.1 `templates` -If you have a lot of repeats in you regexps, you can use `templates` to make your regexps cleaner. +If you have a lot of repeats in your regexps, you can use templates to make your regexps cleaner. You can reference other templates in templates with `!name!`; +For new configurations, prefer centralized templates in `template.yml` / `template.d` via `template_path`. +Inline mapping templates remain supported for backward compatibility. + ```yaml templates: - {name: delimiter, value: '[\s_-]*' } diff --git a/backend/src/api/config_file.rs b/backend/src/api/config_file.rs index 5252bcea1..d983f4f8a 100644 --- a/backend/src/api/config_file.rs +++ b/backend/src/api/config_file.rs @@ -1,13 +1,16 @@ use crate::api::model::{update_app_state_config, update_app_state_sources, AppState, EventMessage}; use crate::model::{Config, SourcesConfig}; use crate::utils; -use crate::utils::{prepare_sources_batch, read_config_file, read_sources_file}; +use crate::utils::{ + read_templates, prepare_sources_batch, read_config_file, read_mappings_file_unprepared, + read_mappings_file_with_templates, read_sources_file, read_sources_file_from_path_with_templates, +}; use arc_swap::access::Access; use arc_swap::ArcSwap; use log::{debug, error, info}; use shared::error::TuliproxError; -use shared::model::{ConfigPaths, ConfigType}; -use std::path::Path; +use shared::model::{ConfigPaths, ConfigType, PatternTemplate}; +use std::path::{Path, PathBuf}; use std::sync::Arc; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -15,17 +18,44 @@ pub enum ConfigFile { Config, ApiProxy, Mapping, + Template, Sources, SourceFile, } impl ConfigFile { - fn load_mapping(app_state: &Arc) -> Result<(), TuliproxError> { + fn load_prepared_global_templates(app_state: &Arc) -> Result>, TuliproxError> { + let paths = > as Access>::load(&app_state.app_config.paths); + let config = > as Access>::load(&app_state.app_config.config); + + let sources_inline_templates = + read_sources_file(paths.sources_file_path.as_str(), false, false, None)?.templates; + let mapping_inline_templates = if let Some(mapping_file_path) = paths.mapping_file_path.as_ref() { + read_mappings_file_unprepared(mapping_file_path, false)? + .map(|(_, mapping)| mapping) + .and_then(|mapping| mapping.mappings.templates) + } else { + None + }; + let template_bundle = read_templates( + paths.template_file_path.as_deref().or(config.template_path.as_deref()), + true, + sources_inline_templates.as_deref(), + mapping_inline_templates.as_deref(), + )?; + Ok(template_bundle.prepared) + } + + fn load_mapping_with_templates( + app_state: &Arc, + prepared_templates: Option<&[PatternTemplate]>, + ) -> Result<(), TuliproxError> { let paths = > as Access>::load(&app_state.app_config.paths); if let Some(mapping_file_path) = paths.mapping_file_path.as_ref() { - match utils::read_mappings(mapping_file_path, true) { + match read_mappings_file_with_templates(mapping_file_path, true, prepared_templates) { Ok(Some((mapping_files, mappings_cfg))) => { - app_state.app_config.set_mappings(mapping_file_path, &mappings_cfg); + let mappings = crate::model::Mappings::from(&mappings_cfg); + app_state.app_config.set_mappings(mapping_file_path, &mappings); for mapping_file in mapping_files { info!("Loaded mapping file {}", mapping_file.display()); } @@ -42,6 +72,11 @@ impl ConfigFile { Ok(()) } + fn load_mapping(app_state: &Arc) -> Result<(), TuliproxError> { + let prepared_templates = Self::load_prepared_global_templates(app_state)?; + Self::load_mapping_with_templates(app_state, prepared_templates.as_deref()) + } + async fn load_api_proxy(app_state: &Arc) -> Result<(), TuliproxError> { match utils::read_api_proxy_config(&app_state.app_config, true).await { Ok(Some(api_proxy)) => { @@ -65,12 +100,38 @@ impl ConfigFile { let paths = > as Access>::load(&app_state.app_config.paths); let config_file = paths.config_file_path.as_str(); let config_dto = read_config_file(config_file, true, true)?; - let mapping_changed = paths.mapping_file_path.as_ref() != config_dto.mapping_path.as_ref(); + + let default_mapping_path = utils::get_default_mappings_path(paths.config_path.as_str()); + let current_mapping_path = paths + .mapping_file_path + .clone() + .unwrap_or_else(|| default_mapping_path.clone()); + let next_mapping_path = config_dto + .mapping_path + .clone() + .filter(|path| !path.trim().is_empty()) + .unwrap_or(default_mapping_path); + let mapping_changed = current_mapping_path != next_mapping_path; + + let default_template_path = utils::get_default_templates_path(paths.config_path.as_str()); + let current_template_path = paths + .template_file_path + .clone() + .unwrap_or_else(|| default_template_path.clone()); + let next_template_path = config_dto + .template_path + .clone() + .filter(|path| !path.trim().is_empty()) + .unwrap_or(default_template_path); + let template_changed = current_template_path != next_template_path; + let mut config: Config = Config::from(config_dto); config.prepare(paths.config_path.as_str()).await?; update_app_state_config(app_state, config).await?; info!("Loaded config file {config_file}"); - if mapping_changed { + if template_changed { + Self::load_sources(app_state).await?; + } else if mapping_changed { Self::load_mapping(app_state)?; } Ok(()) @@ -79,16 +140,23 @@ impl ConfigFile { pub async fn load_sources(app_state: &Arc) -> Result<(), TuliproxError> { let paths = > as Access>::load(&app_state.app_config.paths); let sources_file = paths.sources_file_path.as_str(); + let prepared_templates = Self::load_prepared_global_templates(app_state)?; let mut sources_dto = { let config = > as Access>::load(&app_state.app_config.config); - read_sources_file(sources_file, true, true, config.get_hdhr_device_overview().as_ref())? + read_sources_file_from_path_with_templates( + &PathBuf::from(sources_file), + true, + true, + config.get_hdhr_device_overview().as_ref(), + prepared_templates.as_deref(), + )? }; prepare_sources_batch(&mut sources_dto, true).await?; let sources: SourcesConfig = SourcesConfig::try_from(sources_dto)?; update_app_state_sources(app_state, sources).await?; info!("Loaded sources file {sources_file}"); // mappings are not stored, so we need to reload and apply them if sources change. - Self::load_mapping(app_state) + Self::load_mapping_with_templates(app_state, prepared_templates.as_deref()) } async fn reload_source_file(app_state: &Arc) -> Result<(), TuliproxError> { @@ -107,14 +175,19 @@ impl ConfigFile { ConfigFile::load_mapping(app_state)?; app_state.event_manager.send_event(EventMessage::ConfigChange(ConfigType::Mapping)); } + ConfigFile::Template | ConfigFile::Sources => { + ConfigFile::load_sources(app_state).await?; + let event_type = if matches!(self, ConfigFile::Template) { + ConfigType::Template + } else { + ConfigType::Sources + }; + app_state.event_manager.send_event(EventMessage::ConfigChange(event_type)); + } ConfigFile::Config => { ConfigFile::load_config(app_state).await?; app_state.event_manager.send_event(EventMessage::ConfigChange(ConfigType::Config)); } - ConfigFile::Sources => { - ConfigFile::load_sources(app_state).await?; - app_state.event_manager.send_event(EventMessage::ConfigChange(ConfigType::Sources)); - } ConfigFile::SourceFile => { ConfigFile::reload_source_file(app_state).await?; app_state.event_manager.send_event(EventMessage::ConfigChange(ConfigType::Sources)); diff --git a/backend/src/api/config_watch.rs b/backend/src/api/config_watch.rs index e8b5b6a63..375de528a 100644 --- a/backend/src/api/config_watch.rs +++ b/backend/src/api/config_watch.rs @@ -30,12 +30,23 @@ fn start_config_watch( .mapping_file_path .as_ref() .map_or_else(String::new, ToString::to_string); - let files = get_watch_files(app_state, &paths, mapping_file_path.as_str()); + let template_file_path = paths + .template_file_path + .as_ref() + .map_or_else(String::new, ToString::to_string); + let files = get_watch_files( + app_state, + &paths, + mapping_file_path.as_str(), + template_file_path.as_str(), + ); // // // Add a path to be watched. All files and directories at that path and // // below will be monitored for changes. let path = Path::new(paths.config_path.as_str()); - let recursive_mode = if !mapping_file_path.is_empty() && utils::is_directory(&mapping_file_path) + let recursive_mode = if (!mapping_file_path.is_empty() + && utils::is_directory(&mapping_file_path)) + || (!template_file_path.is_empty() && utils::is_directory(&template_file_path)) { RecursiveMode::Recursive } else { @@ -165,6 +176,7 @@ fn get_watch_files( app_state: &Arc, paths: &ConfigPaths, mapping_file_path: &str, + template_file_path: &str, ) -> HashMap { let sources = > as Access>::load(&app_state.app_config.sources); @@ -174,6 +186,7 @@ fn get_watch_files( (paths.config_file_path.as_str(), ConfigFile::Config), (paths.api_proxy_file_path.as_str(), ConfigFile::ApiProxy), (mapping_file_path, ConfigFile::Mapping), + (template_file_path, ConfigFile::Template), (paths.sources_file_path.as_str(), ConfigFile::Sources), ] .into_iter() diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index e9dfd89f1..9f519d706 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -1,4 +1,5 @@ use crate::api::api_utils::{internal_server_error, try_unwrap_body}; +use crate::api::config_file::ConfigFile; use crate::api::model::AppState; use crate::model::{ApiProxyConfig, InputSource, validate_library_paths_from_dto}; use crate::utils; @@ -64,31 +65,51 @@ async fn save_config_main( async fn save_config_sources( axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(mut sources): axum::extract::Json, + axum::extract::Json(sources): axum::extract::Json, ) -> impl axum::response::IntoResponse + Send { - if let Err(err) = sources.prepare(false, None) { - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response(); - } - - let sources_config = match utils::validate_and_persist_source_config(&app_state, sources).await { + let templates_to_persist = match utils::validate_source_config_for_persist(&app_state, &sources) { Ok(value) => value, Err(err) => { - error!("Failed to save source.yml {err}"); - return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))).into_response(); + error!("Failed to validate source.yml {err}"); + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response(); } }; - // update runtime - match crate::model::SourcesConfig::try_from(&sources_config) { - Ok(src) => { - if let Err(err) = app_state.app_config.set_sources(src) { - return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))).into_response(); - } - app_state.active_provider.update_config(&app_state.app_config).await; - axum::http::StatusCode::OK.into_response() + if let Some(template_definition) = templates_to_persist.as_ref() { + if let Err(err) = utils::persist_templates_config(&app_state, template_definition).await { + error!("Failed to save template config {err}"); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error": err.to_string()})), + ) + .into_response(); } - Err(err) => (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response(), } + + match utils::persist_source_config(&app_state, None, sources).await { + Ok(_) => {} + Err(err) => { + error!("Failed to persist source.yml {err}"); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error": err.to_string()})), + ) + .into_response(); + } + } + + // Reload from disk so runtime always uses fully prepared sources/mappings/templates. + if let Err(err) = ConfigFile::load_sources(&app_state).await { + error!("Failed to reload prepared sources after save {err}"); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error": err.to_string()})), + ) + .into_response(); + } + + app_state.active_provider.update_config(&app_state.app_config).await; + axum::http::StatusCode::OK.into_response() } diff --git a/backend/src/api/main_api.rs b/backend/src/api/main_api.rs index 8d3d3076e..8699c84c8 100644 --- a/backend/src/api/main_api.rs +++ b/backend/src/api/main_api.rs @@ -11,7 +11,11 @@ use crate::api::endpoints::xmltv_api::xmltv_api_register; use crate::api::endpoints::xtream_api::xtream_api_register; use crate::api::hdhomerun_proprietary::spawn_proprietary_tasks; use crate::api::hdhomerun_ssdp::spawn_ssdp_discover_task; -use crate::api::model::{create_cache, create_http_client, create_http_client_no_redirect, ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, DownloadQueue, EventManager, EventMessage, HdHomerunAppState, MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, UpdateGuard}; +use crate::api::model::{ + create_cache, create_http_client, create_http_client_no_redirect, ActiveProviderManager, ActiveUserManager, + AppState, CancelTokens, ConnectionManager, DownloadQueue, EventManager, EventMessage, HdHomerunAppState, + MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, UpdateGuard, +}; use crate::api::panel_api::sync_panel_api_exp_dates_on_boot; use crate::api::scheduler::{exec_interner_prune, exec_scheduler}; use crate::api::serve::serve; @@ -39,12 +43,10 @@ use tokio_util::sync::CancellationToken; use tower_governor::key_extractor::SmartIpKeyExtractor; use tower_http::services::ServeDir; +const METADATA_TRIGGER_WAIT_CYCLE_LIMIT: u32 = 900; + fn get_web_dir_path(web_ui_enabled: bool, web_root: &str) -> Result { - let web_dir_path = if web_root.is_empty() { - get_default_web_root_path() - } else { - PathBuf::from(web_root) - }; + let web_dir_path = if web_root.is_empty() { get_default_web_root_path() } else { PathBuf::from(web_root) }; if web_ui_enabled && (!&web_dir_path.exists() || !&web_dir_path.is_dir()) { return info_err_res!("web_root does not exist or is not a directory: {}", web_dir_path.display()); } @@ -93,7 +95,8 @@ async fn create_shared_data( let active_provider = Arc::new(ActiveProviderManager::new(app_config, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - let connection_manager = Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager)); + let connection_manager = + Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager)); let client = create_http_client(app_config)?; let client_no_redirect = create_http_client_no_redirect(app_config)?; @@ -121,11 +124,7 @@ async fn create_shared_data( }) } -fn exec_update_on_boot( - client: &reqwest::Client, - app_state: &Arc, - targets: &Arc, -) { +fn exec_update_on_boot(client: &reqwest::Client, app_state: &Arc, targets: &Arc) { let cfg = &app_state.app_config; let update_on_boot = { let config = cfg.config.load(); @@ -143,7 +142,20 @@ fn exec_update_on_boot( let event_manager = Some(Arc::clone(&app_state.event_manager)); tokio::spawn(async move { - exec_processing(&client, app_config_clone, targets_clone, event_manager, Some(playlist_state), update_guard, disabled_headers, Some(provider_manager), Some(metadata_manager), None, None).await; + exec_processing( + &client, + app_config_clone, + targets_clone, + event_manager, + Some(playlist_state), + update_guard, + disabled_headers, + Some(provider_manager), + Some(metadata_manager), + None, + None, + ) + .await; }); } } @@ -170,11 +182,7 @@ fn create_cors_layer() -> tower_http::cors::CorsLayer { .max_age(std::time::Duration::from_secs(3600)) } fn create_compression_layer() -> tower_http::compression::CompressionLayer { - tower_http::compression::CompressionLayer::new() - .br(true) - .deflate(true) - .gzip(true) - .zstd(true) + tower_http::compression::CompressionLayer::new().br(true).deflate(true).gzip(true).zstd(true) } pub(in crate::api) fn start_hdhomerun( @@ -190,25 +198,14 @@ pub(in crate::api) fn start_hdhomerun( if hdhomerun.flags.contains(HdHomeRunFlags::Enabled) { if hdhomerun.flags.contains(HdHomeRunFlags::SsdpDiscovery) { info!("HDHomeRun SSDP discovery is enabled."); - spawn_ssdp_discover_task( - Arc::clone(app_config), - host.clone(), - cancel_token.clone(), - ); + spawn_ssdp_discover_task(Arc::clone(app_config), host.clone(), cancel_token.clone()); } else { info!("HDHomeRun SSDP discovery is disabled."); } - if hdhomerun - .flags - .contains(HdHomeRunFlags::ProprietaryDiscovery) - { + if hdhomerun.flags.contains(HdHomeRunFlags::ProprietaryDiscovery) { info!("HDHomeRun proprietary discovery is enabled."); - spawn_proprietary_tasks( - Arc::clone(app_state), - host.clone(), - cancel_token.clone(), - ); + spawn_proprietary_tasks(Arc::clone(app_state), host.clone(), cancel_token.clone()); } else { info!("HDHomeRun proprietary discovery is disabled."); } @@ -220,10 +217,7 @@ pub(in crate::api) fn start_hdhomerun( let port = device.port; let device_clone = Arc::new(device.clone()); let basic_auth = hdhomerun.flags.contains(HdHomeRunFlags::Auth); - infos.push(format!( - "HdHomeRun Server '{}' running: http://{host}:{port}", - device.name - )); + infos.push(format!("HdHomeRun Server '{}' running: http://{host}:{port}", device.name)); let c_token = cancel_token.clone(); let connection_manager = Arc::clone(&app_data.connection_manager); tokio::spawn(async move { @@ -232,16 +226,13 @@ pub(in crate::api) fn start_hdhomerun( .layer(create_compression_layer()) .merge(hdhr_api_register(basic_auth)); - let router: axum::Router<()> = - router.with_state(Arc::new(HdHomerunAppState { - app_state: Arc::clone(&app_data), - device: Arc::clone(&device_clone), - hd_scan_state: Arc::new(AtomicI8::new(-1)), - })); + let router: axum::Router<()> = router.with_state(Arc::new(HdHomerunAppState { + app_state: Arc::clone(&app_data), + device: Arc::clone(&device_clone), + hd_scan_state: Arc::new(AtomicI8::new(-1)), + })); - match tokio::net::TcpListener::bind(format!("{}:{}", app_host.clone(), port)) - .await - { + match tokio::net::TcpListener::bind(format!("{}:{}", app_host.clone(), port)).await { Ok(listener) => { serve(listener, router, Some(c_token), &connection_manager).await; } @@ -255,10 +246,7 @@ pub(in crate::api) fn start_hdhomerun( } #[allow(clippy::too_many_lines)] -pub async fn start_server( - app_config: Arc, - targets: Arc, -) -> Result<(), TuliproxError> { +pub async fn start_server(app_config: Arc, targets: Arc) -> Result<(), TuliproxError> { let mut infos = Vec::new(); let cfg = app_config.config.load(); let host = cfg.api.host.clone(); @@ -289,11 +277,7 @@ pub async fn start_server( let (cancel_token_scheduler, cancel_token_hdhomerun, cancel_token_file_watch) = { let cancel_tokens = app_state.cancel_tokens.load(); - ( - cancel_tokens.scheduler.clone(), - cancel_tokens.hdhomerun.clone(), - cancel_tokens.file_watch.clone(), - ) + (cancel_tokens.scheduler.clone(), cancel_tokens.hdhomerun.clone(), cancel_tokens.file_watch.clone()) }; if let Err(err) = load_playlists_into_memory_cache(&app_state).await { @@ -306,18 +290,9 @@ pub async fn start_server( sync_panel_api_exp_dates_on_boot(&app_state).await; - exec_scheduler( - client.as_ref(), - &app_state, - &targets, - &cancel_token_scheduler, - ); + exec_scheduler(client.as_ref(), &app_state, &targets, &cancel_token_scheduler); - exec_update_on_boot( - client.as_ref(), - &app_state, - &targets, - ); + exec_update_on_boot(client.as_ref(), &app_state, &targets); exec_file_lock_prune(&app_state); @@ -331,12 +306,7 @@ pub async fn start_server( start_hdhomerun(&app_config, &app_state, &mut infos, &cancel_token_hdhomerun); } - let web_ui_path = cfg - .web_ui - .as_ref() - .and_then(|c| c.path.as_ref()) - .cloned() - .unwrap_or_default(); + let web_ui_path = cfg.web_ui.as_ref().and_then(|c| c.path.as_ref()).cloned().unwrap_or_default(); infos.push(format!("Server running: http://{}:{}", &cfg.api.host, &cfg.api.port)); for info in &infos { info!("{info}"); @@ -346,10 +316,7 @@ pub async fn start_server( let mut router = axum::Router::new() .route("/healthcheck", axum::routing::get(healthcheck)) .nest_service("/.well-known", ServeDir::new(web_dir_path.join("static/.well-known"))) - .merge(ws_api_register( - web_auth_enabled, - web_ui_path.as_str(), - )); + .merge(ws_api_register(web_auth_enabled, web_ui_path.as_str())); if web_ui_enabled { router = router .nest_service( @@ -360,16 +327,9 @@ pub async fn start_server( &concat_path_leading_slash(&web_ui_path, "assets"), tower_http::services::ServeDir::new(web_dir_path.join("assets")), ) - .merge(v1_api_register( - web_auth_enabled, - Arc::clone(&shared_data), - web_ui_path.as_str(), - )); + .merge(v1_api_register(web_auth_enabled, Arc::clone(&shared_data), web_ui_path.as_str())); if !web_ui_path.is_empty() { - router = router.merge(index_register_with_path( - &web_dir_path, - web_ui_path.as_str(), - )); + router = router.merge(index_register_with_path(&web_dir_path, web_ui_path.as_str())); } } @@ -379,11 +339,7 @@ pub async fn start_server( .merge(xmltv_api_register()) .merge(hls_api_register()) .merge(cvs_api_register()); - if let Some(rate_limiter) = cfg - .reverse_proxy - .as_ref() - .and_then(|r| r.rate_limit.clone()) - { + if let Some(rate_limiter) = cfg.reverse_proxy.as_ref().and_then(|r| r.rate_limit.clone()) { api_router = add_rate_limiter(api_router, &rate_limiter); } @@ -393,21 +349,18 @@ pub async fn start_server( router = router.merge(index_register_without_path(&web_dir_path)); } - router = router - .layer(axum::middleware::from_fn(log_req)) - .layer(create_cors_layer()) - .layer(create_compression_layer()); + router = + router.layer(axum::middleware::from_fn(log_req)).layer(create_cors_layer()).layer(create_compression_layer()); let router: axum::Router<()> = router.with_state(shared_data.clone()); - let listener = tokio::net::TcpListener::bind(format!("{host}:{port}")).await.map_err(|err| info_err!("Failed to bind to {host}:{port}, {err}"))?; + let listener = tokio::net::TcpListener::bind(format!("{host}:{port}")) + .await + .map_err(|err| info_err!("Failed to bind to {host}:{port}, {err}"))?; serve(listener, router, None, &shared_data.connection_manager).await; Ok(()) } -fn add_rate_limiter( - router: Router>, - rate_limit_cfg: &RateLimitConfig, -) -> Router> { +fn add_rate_limiter(router: Router>, rate_limit_cfg: &RateLimitConfig) -> Router> { if rate_limit_cfg.enabled { let governor_conf = tower_governor::governor::GovernorConfigBuilder::default() .key_extractor(SmartIpKeyExtractor) @@ -448,14 +401,9 @@ async fn log_req(req: Request, next: Next) -> impl axum::response::IntoResponse .filter(|v| !v.is_empty()) .map(str::to_string) }) - .or_else(|| { - req.extensions() - .get::>() - .map(|c| c.0.to_string()) - }); + .or_else(|| req.extensions().get::>().map(|c| c.0.to_string())); - let safe_ip = client_ip - .as_deref().map_or_else(|| sanitize_sensitive_info(""), sanitize_sensitive_info); + let safe_ip = client_ip.as_deref().map_or_else(|| sanitize_sensitive_info(""), sanitize_sensitive_info); let uri_string = uri.to_string(); let safe_uri = sanitize_sensitive_info(&uri_string); @@ -463,7 +411,7 @@ async fn log_req(req: Request, next: Next) -> impl axum::response::IntoResponse next.run(req).await } - +#[allow(clippy::too_many_lines)] fn exec_input_update_listener(app_state: &Arc, targets: &Arc) { let app_state = Arc::clone(app_state); let targets = Arc::clone(targets); @@ -503,7 +451,9 @@ fn exec_input_update_listener(app_state: &Arc, targets: &Arc, targets: &Arc= METADATA_TRIGGER_WAIT_CYCLE_LIMIT { + warn!( + "Aborting metadata-triggered update after waiting ~{}s for playlist lock ({} cycles)", + wait_cycles * 2, + wait_cycles + ); + break; + } + if wait_cycles.is_multiple_of(30) { + debug!( + "Metadata-triggered update is still waiting for active playlist update to finish (waited ~{}s)", + wait_cycles * 2 + ); + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; } }); } else { diff --git a/backend/src/api/model/active_provider_manager.rs b/backend/src/api/model/active_provider_manager.rs index e748eb4d9..f0860a1d9 100644 --- a/backend/src/api/model/active_provider_manager.rs +++ b/backend/src/api/model/active_provider_manager.rs @@ -298,7 +298,8 @@ impl ActiveProviderManager { &self, input_name: &Arc, ) -> Option { - self.acquire_connection_inner(input_name, &DUMMY_ADDR, false, None, DEFAULT_PROBE_PRIORITY).await + // Probe is strictly low-priority and must never consume grace capacity. + self.acquire_connection_inner(input_name, &DUMMY_ADDR, false, Some(false), DEFAULT_PROBE_PRIORITY).await } // This method is used for redirects to cycle through the provider @@ -530,4 +531,4 @@ impl ActiveProviderManager { pub async fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count().await } -} \ No newline at end of file +} diff --git a/backend/src/api/model/app_state.rs b/backend/src/api/model/app_state.rs index a8228e49f..4a31810e3 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/src/api/model/app_state.rs @@ -429,6 +429,10 @@ impl AppState { string_changed, old_config.mapping_path.as_ref(), config.mapping_path.as_ref() + ) || change_detect!( + string_changed, + old_config.template_path.as_ref(), + config.template_path.as_ref() ); let geoip_enabled = config.is_geoip_enabled(); diff --git a/backend/src/api/model/metadata_update_manager.rs b/backend/src/api/model/metadata_update_manager.rs index ccb35529f..268a550fd 100644 --- a/backend/src/api/model/metadata_update_manager.rs +++ b/backend/src/api/model/metadata_update_manager.rs @@ -1,24 +1,34 @@ use crate::api::model::ProviderHandle; use crate::api::model::{AppState, EventMessage}; -use crate::processing::processor::{update_generic_stream_metadata, update_live_stream_metadata, update_series_metadata, update_vod_metadata}; +use crate::processing::processor::{ + update_generic_stream_metadata, update_live_stream_metadata, update_series_metadata, update_vod_metadata, +}; use crate::utils::debug_if_enabled; use dashmap::mapref::entry::Entry; use dashmap::DashMap; use log::{debug, error, info, warn}; +use parking_lot::Mutex as ParkingMutex; use shared::create_bitset; use shared::error::TuliproxError; -use shared::model::{InputType, LiveStreamProperties, PlaylistItemType, SeriesStreamProperties, UUIDType, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}; +use shared::model::{ + InputType, LiveStreamProperties, PlaylistItemType, SeriesStreamProperties, UUIDType, VideoStreamProperties, + XtreamCluster, XtreamPlaylistItem, +}; use shared::utils::generate_playlist_uuid; use std::sync::atomic::{AtomicU64, Ordering}; -use parking_lot::Mutex as ParkingMutex; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{mpsc, Mutex, RwLock}; use tokio_util::sync::CancellationToken; use crate::api::model::BatchResultCollector; -use crate::repository::{get_input_storage_path, get_target_id_mapping_file, write_playlist_batch_item_upsert, xtream_get_file_path, BPlusTreeQuery}; -use crate::repository::{persist_input_live_info_batch, persist_input_series_info_batch, persist_input_vod_info_batch, TargetIdMapping}; +use crate::repository::{ + get_input_storage_path, get_target_id_mapping_file, write_playlist_batch_item_upsert, xtream_get_file_path, + BPlusTreeQuery, +}; +use crate::repository::{ + persist_input_live_info_batch, persist_input_series_info_batch, persist_input_vod_info_batch, TargetIdMapping, +}; use crate::utils::FileReadGuard; use std::cmp::min; use std::collections::{HashMap, HashSet}; @@ -26,10 +36,13 @@ use std::collections::{HashMap, HashSet}; const QUEUE_LOG_INTERVAL: Duration = Duration::from_secs(30); const PROGRESS_LOG_INTERVAL: Duration = Duration::from_secs(15); const MAX_RETRY_BACKOFF_SECS: u64 = 60; +const UPDATE_IN_PROGRESS_RETRY_SECS: u64 = 2; +const MIN_PROBE_RETRY_BASE_SECS: u16 = 5; const MAX_QUEUE_SIZE: usize = 100_000; const TASK_ERR_NO_CONNECTION: &str = "No connection available"; const TASK_ERR_PREEMPTED: &str = "Task preempted"; +const TASK_ERR_UPDATE_IN_PROGRESS: &str = "Playlist update in progress"; create_bitset!(u8, ResolveReason, Info, Tmdb, Date, Probe, MissingDetails); @@ -69,20 +82,40 @@ impl From for ProviderIdType { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum UpdateTask { - ResolveVod { id: ProviderIdType, reason: ResolveReasonSet, delay: u16 }, - ResolveSeries { id: ProviderIdType, reason: ResolveReasonSet, delay: u16 }, - ProbeLive { id: ProviderIdType, reason: ResolveReasonSet, delay: u16, interval: u64 }, + ResolveVod { + id: ProviderIdType, + reason: ResolveReasonSet, + delay: u16, + }, + ResolveSeries { + id: ProviderIdType, + reason: ResolveReasonSet, + delay: u16, + }, + ProbeLive { + id: ProviderIdType, + reason: ResolveReasonSet, + delay: u16, + interval: u64, + }, // Generic probe for M3U/Library/etc. - ProbeStream { probe_scope: Arc, unique_id: String, url: String, item_type: PlaylistItemType, reason: ResolveReasonSet, delay: u16 }, + ProbeStream { + probe_scope: Arc, + unique_id: String, + url: String, + item_type: PlaylistItemType, + reason: ResolveReasonSet, + delay: u16, + }, } impl UpdateTask { pub fn delay(&self) -> u16 { match self { - UpdateTask::ResolveVod { delay, .. } | - UpdateTask::ResolveSeries { delay, .. } | - UpdateTask::ProbeLive { delay, .. } | - UpdateTask::ProbeStream { delay, .. } => *delay, + UpdateTask::ResolveVod { delay, .. } + | UpdateTask::ResolveSeries { delay, .. } + | UpdateTask::ProbeLive { delay, .. } + | UpdateTask::ProbeStream { delay, .. } => *delay, } } } @@ -90,10 +123,18 @@ impl UpdateTask { impl std::fmt::Display for UpdateTask { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - UpdateTask::ResolveVod { id, reason, delay } => write!(f, "Resolve VOD {id} (Reason: {reason}, Delay: {delay}sec)"), - UpdateTask::ResolveSeries { id, reason, delay } => write!(f, "Resolve Series {id} (Reason: {reason}, Delay: {delay}sec)"), - UpdateTask::ProbeLive { id, reason, delay, interval } => write!(f, "Probe Live {id} (Reason: {reason}, Delay: {delay}sec, Interval: {interval}secs )"), - UpdateTask::ProbeStream { probe_scope, unique_id, reason, delay, .. } => write!(f, "Probe Stream {probe_scope}/{unique_id} (Reason: {reason}, Delay: {delay}sec)"), + UpdateTask::ResolveVod { id, reason, delay } => { + write!(f, "Resolve VOD {id} (Reason: {reason}, Delay: {delay}sec)") + } + UpdateTask::ResolveSeries { id, reason, delay } => { + write!(f, "Resolve Series {id} (Reason: {reason}, Delay: {delay}sec)") + } + UpdateTask::ProbeLive { id, reason, delay, interval } => { + write!(f, "Probe Live {id} (Reason: {reason}, Delay: {delay}sec, Interval: {interval}secs )") + } + UpdateTask::ProbeStream { probe_scope, unique_id, reason, delay, .. } => { + write!(f, "Probe Stream {probe_scope}/{unique_id} (Reason: {reason}, Delay: {delay}sec)") + } } } } @@ -150,10 +191,7 @@ struct PendingTask { impl PendingTask { fn new(task: UpdateTask) -> Self { - Self { - task: Mutex::new(task), - generation: AtomicU64::new(0), - } + Self { task: Mutex::new(task), generation: AtomicU64::new(0) } } } @@ -170,6 +208,11 @@ pub struct MetadataUpdateManager { workers: DashMap, InputWorkerContext>, /// Global application state (weak reference to avoid cycles) app_state: tokio::sync::Mutex>>, + /// Global gate: + /// - Foreground playlist updates hold WRITE lock. + /// - Background metadata/probe tasks hold READ lock per task. + /// This guarantees that no background task runs while an update is active. + update_pause_gate: Arc>, /// Global cancellation token for shutdown cancel_token: CancellationToken, } @@ -185,10 +228,17 @@ impl MetadataUpdateManager { Self { workers: DashMap::new(), app_state: tokio::sync::Mutex::new(None), + update_pause_gate: Arc::new(RwLock::new(())), cancel_token, } } + /// Acquire exclusive gate for a foreground playlist update. + /// While this guard is held, background metadata/probe tasks are paused. + pub async fn acquire_update_pause_guard(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { + self.update_pause_gate.clone().write_owned().await + } + pub async fn set_app_state(&self, app_state: Weak) { let mut guard = self.app_state.lock().await; *guard = Some(app_state); @@ -226,10 +276,7 @@ impl MetadataUpdateManager { let (tx, rx) = mpsc::channel::(256); let pending_tasks = Arc::new(DashMap::new()); - let ctx = InputWorkerContext { - sender: tx.clone(), - pending_tasks: pending_tasks.clone(), - }; + let ctx = InputWorkerContext { sender: tx.clone(), pending_tasks: pending_tasks.clone() }; entry.insert(ctx.clone()); worker_to_spawn = Some(InputWorker { @@ -238,6 +285,7 @@ impl MetadataUpdateManager { receiver: rx, pending_tasks, app_state_weak, + update_pause_gate: Arc::clone(&self.update_pause_gate), cancel_token: self.cancel_token.clone(), batch_buffer: BatchResultCollector::new(), db_handles: HashMap::new(), @@ -274,14 +322,26 @@ impl MetadataUpdateManager { let mut merged = false; // Merge logic match (&mut *existing, task) { - (UpdateTask::ResolveVod { reason: r1, delay: d1, .. }, UpdateTask::ResolveVod { reason: r2, delay: d2, .. }) - | (UpdateTask::ResolveSeries { reason: r1, delay: d1, .. }, UpdateTask::ResolveSeries { reason: r2, delay: d2, .. }) - | (UpdateTask::ProbeStream { reason: r1, delay: d1, .. }, UpdateTask::ProbeStream { reason: r2, delay: d2, .. }) => { + ( + UpdateTask::ResolveVod { reason: r1, delay: d1, .. }, + UpdateTask::ResolveVod { reason: r2, delay: d2, .. }, + ) + | ( + UpdateTask::ResolveSeries { reason: r1, delay: d1, .. }, + UpdateTask::ResolveSeries { reason: r2, delay: d2, .. }, + ) + | ( + UpdateTask::ProbeStream { reason: r1, delay: d1, .. }, + UpdateTask::ProbeStream { reason: r2, delay: d2, .. }, + ) => { *r1 |= r2; *d1 = min(*d1, d2); merged = true; } - (UpdateTask::ProbeLive { reason: r1, delay: d1, interval: i1, .. }, UpdateTask::ProbeLive { reason: r2, delay: d2, interval: i2, .. }) => { + ( + UpdateTask::ProbeLive { reason: r1, delay: d1, interval: i1, .. }, + UpdateTask::ProbeLive { reason: r2, delay: d2, interval: i2, .. }, + ) => { *r1 |= r2; *d1 = min(*d1, d2); *i1 = min(*i1, i2); @@ -331,6 +391,7 @@ struct InputWorker { receiver: mpsc::Receiver, pending_tasks: Arc>, app_state_weak: Option>, + update_pause_gate: Arc>, cancel_token: CancellationToken, batch_buffer: BatchResultCollector, db_handles: HashMap, @@ -356,11 +417,7 @@ impl InputWorker { let mut next_task: Option<(TaskKey, UpdateTask, u64)> = None; loop { - let task_data = if let Some(t) = next_task.take() { - Some(t) - } else { - self.recv_task_fast_or_wait().await - }; + let task_data = if let Some(t) = next_task.take() { Some(t) } else { self.recv_task_fast_or_wait().await }; let Some((current_key, current_task, current_generation)) = task_data else { break }; @@ -375,9 +432,7 @@ impl InputWorker { .checked_sub(QUEUE_LOG_INTERVAL + Duration::from_secs(1)) .unwrap_or_else(Instant::now); if let Some(app_state) = app_state_weak.as_ref().and_then(Weak::upgrade) { - app_state - .event_manager - .send_event(EventMessage::InputMetadataUpdatesStarted(input_name.clone())); + app_state.event_manager.send_event(EventMessage::InputMetadataUpdatesStarted(input_name.clone())); } debug!("Background metadata update queue has entries for input {input_name}; starting processing"); } @@ -387,16 +442,22 @@ impl InputWorker { let mut requeue_current = false; let mut retry_delay_secs = 0_u64; - match Self::process_task_static( - &input_name, - app_state_weak.as_ref(), - ¤t_task, - &mut self.batch_buffer, - &mut self.db_handles, - &mut self.failed_clusters, - ) + // Hold READ gate while executing one background task. + // Foreground updates acquire WRITE gate and therefore pause this path. + let task_result = { + let _task_gate_guard = self.update_pause_gate.clone().read_owned().await; + Self::process_task_static( + &input_name, + app_state_weak.as_ref(), + ¤t_task, + &mut self.batch_buffer, + &mut self.db_handles, + &mut self.failed_clusters, + ) .await - { + }; + + match task_result { Ok(task_changed) => { if Self::is_vod_task_key(¤t_key) { processed_vod_count += 1; @@ -408,8 +469,7 @@ impl InputWorker { if last_progress_log_at.elapsed() >= PROGRESS_LOG_INTERVAL { // current_key is removed from pending_tasks later in this loop iteration; // subtract it here so "remaining" reflects the post-success queue size. - let (mut remaining_vod, mut remaining_series) = - Self::queue_resolve_counts(&self.pending_tasks); + let (mut remaining_vod, mut remaining_series) = Self::queue_resolve_counts(&self.pending_tasks); if Self::is_vod_task_key(¤t_key) { remaining_vod = remaining_vod.saturating_sub(1); } else if Self::is_series_task_key(¤t_key) { @@ -435,22 +495,51 @@ impl InputWorker { // If this is the only queued signal, use exponential backoff. // Otherwise, push to the back immediately to improve throughput/fairness. if self.receiver.is_empty() { - retry_delay_secs = Self::compute_retry_backoff_secs(current_task.delay(), attempts); + retry_delay_secs = if Self::is_probe_task(¤t_task) { + Self::compute_probe_retry_backoff_secs(current_task.delay(), attempts) + } else { + Self::compute_retry_backoff_secs(current_task.delay(), attempts) + }; } - debug_if_enabled!("No provider connection for task {} on input {}, requeueing (attempt={}, retry_delay={}s, queue_len={})", - current_task, - &input_name, - attempts, - retry_delay_secs, - self.receiver.len() - ); + if Self::is_probe_task(¤t_task) { + debug_if_enabled!( + "No provider connection for low-priority probe task {} on input {}, requeueing (attempt={}, retry_delay={}s, queue_len={})", + current_task, + &input_name, + attempts, + retry_delay_secs, + self.receiver.len() + ); + } else { + debug_if_enabled!("No provider connection for task {} on input {}, requeueing (attempt={}, retry_delay={}s, queue_len={})", + current_task, + &input_name, + attempts, + retry_delay_secs, + self.receiver.len() + ); + } + } + Err(e) if e.message == TASK_ERR_UPDATE_IN_PROGRESS => { + requeue_current = true; + retry_attempts.remove(¤t_key); + let update_completed = + Self::wait_for_playlist_update_completion(app_state_weak.as_ref(), &self.cancel_token).await; + if !update_completed { + // Fallback path (e.g. listener closed): keep retrying with a small delay. + retry_delay_secs = UPDATE_IN_PROGRESS_RETRY_SECS; + } + } + Err(e) if e.message == TASK_ERR_PREEMPTED => { + // Preempted by a user-facing request: defer and retry later. + requeue_current = true; + retry_attempts.remove(¤t_key); + retry_delay_secs = UPDATE_IN_PROGRESS_RETRY_SECS; } Err(e) => { retry_attempts.remove(¤t_key); - if e.message != TASK_ERR_PREEMPTED { - error!("Task {current_task} failed for input {input_name}: {e}"); - } + error!("Task {current_task} failed for input {input_name}: {e}"); } } @@ -474,9 +563,7 @@ impl InputWorker { warn!("Failed to requeue task {current_task} for input {input_name}"); } } else { - self - .finalize_processed_task_success(¤t_key, current_generation, &input_name) - .await; + self.finalize_processed_task_success(¤t_key, current_generation, &input_name).await; // Rate limiting if delay_secs > 0 @@ -499,14 +586,16 @@ impl InputWorker { } } - let queue_has_work = - next_task.is_some() || !self.receiver.is_empty() || !self.pending_tasks.is_empty(); + let queue_has_work = next_task.is_some() || !self.receiver.is_empty() || !self.pending_tasks.is_empty(); if queue_has_work { // Avoid O(n) queue scans per task; report queue status periodically. if last_queue_log_at.elapsed() >= QUEUE_LOG_INTERVAL { let queue_counts = Self::queue_resolve_counts(&self.pending_tasks); - debug!("In queue to resolve vod: {}, series: {} (input: {input_name})", queue_counts.0, queue_counts.1); + debug!( + "In queue to resolve vod: {}, series: {} (input: {input_name})", + queue_counts.0, queue_counts.1 + ); last_queue_log_at = Instant::now(); } } else { @@ -604,9 +693,49 @@ impl InputWorker { fn compute_retry_backoff_secs(base_delay_secs: u16, attempts: u8) -> u64 { let base_delay = u64::from(base_delay_secs.max(1)); let exp = u32::from(attempts.saturating_sub(1).min(6)); - base_delay - .saturating_mul(2_u64.saturating_pow(exp)) - .min(MAX_RETRY_BACKOFF_SECS) + base_delay.saturating_mul(2_u64.saturating_pow(exp)).min(MAX_RETRY_BACKOFF_SECS) + } + + fn compute_probe_retry_backoff_secs(base_delay_secs: u16, attempts: u8) -> u64 { + let adjusted_base = base_delay_secs.max(MIN_PROBE_RETRY_BASE_SECS); + Self::compute_retry_backoff_secs(adjusted_base, attempts) + } + + async fn wait_for_playlist_update_completion( + app_state_weak: Option<&Weak>, + cancel_token: &CancellationToken, + ) -> bool { + let Some(app_state) = app_state_weak.and_then(Weak::upgrade) else { + return false; + }; + + // Subscribe before checking the guard to avoid missing a PlaylistUpdate event + // published between the guard check and receiver subscription. + let mut rx = app_state.event_manager.get_event_channel(); + + // If no update is active anymore, do not block on events. + if let Some(guard) = app_state.update_guard.try_playlist() { + drop(guard); + return true; + } + + loop { + tokio::select! { + () = cancel_token.cancelled() => return false, + recv = rx.recv() => { + match recv { + Ok(EventMessage::PlaylistUpdate(_)) => return true, + Ok(_) | Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + if let Some(guard) = app_state.update_guard.try_playlist() { + drop(guard); + return true; + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => return false, + } + } + } + } } async fn sleep_or_cancel(cancel_token: &CancellationToken, duration: Duration) -> bool { @@ -641,13 +770,20 @@ impl InputWorker { matches!(key, TaskKey::Series(_) | TaskKey::SeriesStr(_)) } + #[inline] + fn is_probe_task(task: &UpdateTask) -> bool { + matches!(task, UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. }) + } + // Changed to static method async fn flush_batch_static( input_name: &str, app_state_weak: Option<&Weak>, batch_buffer: &mut BatchResultCollector, ) { - if batch_buffer.is_empty() { return; } + if batch_buffer.is_empty() { + return; + } let Some(app_state) = app_state_weak.and_then(Weak::upgrade) else { return }; let app_config = &app_state.app_config; @@ -670,7 +806,15 @@ impl InputWorker { } if !updates.is_empty() { - if let Err(e) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, input_name, updates).await { + if let Err(e) = persist_input_vod_info_batch( + app_config, + &storage_path, + XtreamCluster::Video, + input_name, + updates, + ) + .await + { error!("Failed to flush VOD batch for input {input_name}: {e}"); } } @@ -685,7 +829,15 @@ impl InputWorker { } if !updates.is_empty() { - if let Err(e) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, input_name, updates).await { + if let Err(e) = persist_input_series_info_batch( + app_config, + &storage_path, + XtreamCluster::Series, + input_name, + updates, + ) + .await + { error!("Failed to flush Series batch for input {input_name}: {e}"); } } @@ -700,25 +852,24 @@ impl InputWorker { } if !updates.is_empty() { - if let Err(e) = persist_input_live_info_batch(app_config, &storage_path, XtreamCluster::Live, input_name, updates).await { + if let Err(e) = persist_input_live_info_batch( + app_config, + &storage_path, + XtreamCluster::Live, + input_name, + updates, + ) + .await + { error!("Failed to flush Live batch for input {input_name}: {e}"); } } } } - let cascade_batch = BatchResultCollector { - vod: vod_updates, - series: series_updates, - live: live_updates, - }; + let cascade_batch = BatchResultCollector { vod: vod_updates, series: series_updates, live: live_updates }; - Self::cascade_updates( - &app_state, - &app_config.config.load(), - input_name, - &cascade_batch, - ).await; + Self::cascade_updates(&app_state, &app_config.config.load(), input_name, &cascade_batch).await; } #[allow(clippy::too_many_lines)] @@ -747,12 +898,18 @@ impl InputWorker { affected_targets }; - if targets.is_empty() { return; } + if targets.is_empty() { + return; + } for target in targets { let target_name = &target.name; - let Some(target_path) = crate::repository::get_target_storage_path(config, target_name) else { continue; }; - let Some(storage_path) = crate::repository::xtream_get_storage_path(config, target_name) else { continue; }; + let Some(target_path) = crate::repository::get_target_storage_path(config, target_name) else { + continue; + }; + let Some(storage_path) = crate::repository::xtream_get_storage_path(config, target_name) else { + continue; + }; let mapping_file = get_target_id_mapping_file(&target_path); // Read mapping under lock inside spawn_blocking to avoid blocking async executor. @@ -785,12 +942,7 @@ impl InputWorker { &mut provider_virtual_ids, &mut uuid_virtual_ids, ); - Self::apply_vod_cascade_updates( - app_state, - &target, - &storage_path, - vod_virtual_updates, - ).await; + Self::apply_vod_cascade_updates(app_state, &target, &storage_path, vod_virtual_updates).await; let series_virtual_updates = Self::collect_series_virtual_updates( &mapping, @@ -799,12 +951,7 @@ impl InputWorker { &mut provider_virtual_ids, &mut uuid_virtual_ids, ); - Self::apply_series_cascade_updates( - app_state, - &target, - &storage_path, - series_virtual_updates, - ).await; + Self::apply_series_cascade_updates(app_state, &target, &storage_path, series_virtual_updates).await; let live_virtual_updates = Self::collect_live_virtual_updates( &mapping, @@ -813,12 +960,7 @@ impl InputWorker { &mut provider_virtual_ids, &mut uuid_virtual_ids, ); - Self::apply_live_cascade_updates( - app_state, - &target, - &storage_path, - live_virtual_updates, - ).await; + Self::apply_live_cascade_updates(app_state, &target, &storage_path, live_virtual_updates).await; } } @@ -864,9 +1006,7 @@ impl InputWorker { PlaylistItemType::Video, props.direct_source.as_ref(), ); - if let Some(virtual_id) = - Self::get_cached_uuid_virtual_id(mapping, uuid_virtual_ids, uuid) - { + if let Some(virtual_id) = Self::get_cached_uuid_virtual_id(mapping, uuid_virtual_ids, uuid) { virtual_updates.insert(virtual_id, props); } } @@ -899,15 +1039,8 @@ impl InputWorker { } } ProviderIdType::Text(provider_id_text) => { - let uuid = generate_playlist_uuid( - input_name, - provider_id_text, - PlaylistItemType::Series, - "", - ); - if let Some(virtual_id) = - Self::get_cached_uuid_virtual_id(mapping, uuid_virtual_ids, uuid) - { + let uuid = generate_playlist_uuid(input_name, provider_id_text, PlaylistItemType::Series, ""); + if let Some(virtual_id) = Self::get_cached_uuid_virtual_id(mapping, uuid_virtual_ids, uuid) { virtual_updates.insert(virtual_id, props); } } @@ -946,9 +1079,7 @@ impl InputWorker { PlaylistItemType::Live, props.direct_source.as_ref(), ); - if let Some(virtual_id) = - Self::get_cached_uuid_virtual_id(mapping, uuid_virtual_ids, uuid) - { + if let Some(virtual_id) = Self::get_cached_uuid_virtual_id(mapping, uuid_virtual_ids, uuid) { virtual_updates.insert(virtual_id, props); } } @@ -970,10 +1101,8 @@ impl InputWorker { let target_name = target.name.as_str(); let xtream_path = xtream_get_file_path(storage_path, XtreamCluster::Video); - let updates_input: Vec<(u32, VideoStreamProperties)> = virtual_updates - .into_iter() - .map(|(vid, props)| (vid, props.clone())) - .collect(); + let updates_input: Vec<(u32, VideoStreamProperties)> = + virtual_updates.into_iter().map(|(vid, props)| (vid, props.clone())).collect(); let file_lock = app_state.app_config.file_locks.read_lock(&xtream_path).await; let xtream_path = xtream_path.clone(); @@ -985,13 +1114,14 @@ impl InputWorker { }; for (virtual_id, props) in updates_input { if let Ok(Some(mut item)) = query.query_zero_copy(&virtual_id) { - item.additional_properties = - Some(shared::model::StreamProperties::Video(Box::new(props))); + item.additional_properties = Some(shared::model::StreamProperties::Video(Box::new(props))); updates.push(item); } } updates - }).await { + }) + .await + { Ok(updates) => updates, Err(err) => { error!("Failed to read VOD updates from disk for {target_name}: {err}"); @@ -1003,12 +1133,9 @@ impl InputWorker { return; } - if let Err(e) = write_playlist_batch_item_upsert( - &app_state.app_config, - target_name, - XtreamCluster::Video, - &updates, - ).await { + if let Err(e) = + write_playlist_batch_item_upsert(&app_state.app_config, target_name, XtreamCluster::Video, &updates).await + { error!("Failed to cascade VOD updates to target {target_name}: {e}"); return; } @@ -1030,10 +1157,8 @@ impl InputWorker { let target_name = target.name.as_str(); let xtream_path = xtream_get_file_path(storage_path, XtreamCluster::Series); - let updates_input: Vec<(u32, SeriesStreamProperties)> = virtual_updates - .into_iter() - .map(|(vid, props)| (vid, props.clone())) - .collect(); + let updates_input: Vec<(u32, SeriesStreamProperties)> = + virtual_updates.into_iter().map(|(vid, props)| (vid, props.clone())).collect(); let file_lock = app_state.app_config.file_locks.read_lock(&xtream_path).await; let xtream_path = xtream_path.clone(); @@ -1045,13 +1170,14 @@ impl InputWorker { }; for (virtual_id, props) in updates_input { if let Ok(Some(mut item)) = query.query_zero_copy(&virtual_id) { - item.additional_properties = - Some(shared::model::StreamProperties::Series(Box::new(props))); + item.additional_properties = Some(shared::model::StreamProperties::Series(Box::new(props))); updates.push(item); } } updates - }).await { + }) + .await + { Ok(updates) => updates, Err(err) => { error!("Failed to read Series updates from disk for {target_name}: {err}"); @@ -1063,12 +1189,9 @@ impl InputWorker { return; } - if let Err(e) = write_playlist_batch_item_upsert( - &app_state.app_config, - target_name, - XtreamCluster::Series, - &updates, - ).await { + if let Err(e) = + write_playlist_batch_item_upsert(&app_state.app_config, target_name, XtreamCluster::Series, &updates).await + { error!("Failed to cascade Series updates to target {target_name}: {e}"); return; } @@ -1090,10 +1213,8 @@ impl InputWorker { let target_name = target.name.as_str(); let xtream_path = xtream_get_file_path(storage_path, XtreamCluster::Live); - let updates_input: Vec<(u32, LiveStreamProperties)> = virtual_updates - .into_iter() - .map(|(vid, props)| (vid, props.clone())) - .collect(); + let updates_input: Vec<(u32, LiveStreamProperties)> = + virtual_updates.into_iter().map(|(vid, props)| (vid, props.clone())).collect(); let file_lock = app_state.app_config.file_locks.read_lock(&xtream_path).await; let xtream_path = xtream_path.clone(); @@ -1105,13 +1226,14 @@ impl InputWorker { }; for (virtual_id, props) in updates_input { if let Ok(Some(mut item)) = query.query_zero_copy(&virtual_id) { - item.additional_properties = - Some(shared::model::StreamProperties::Live(Box::new(props))); + item.additional_properties = Some(shared::model::StreamProperties::Live(Box::new(props))); updates.push(item); } } updates - }).await { + }) + .await + { Ok(updates) => updates, Err(err) => { error!("Failed to read Live updates from disk for {target_name}: {err}"); @@ -1123,12 +1245,9 @@ impl InputWorker { return; } - if let Err(e) = write_playlist_batch_item_upsert( - &app_state.app_config, - target_name, - XtreamCluster::Live, - &updates, - ).await { + if let Err(e) = + write_playlist_batch_item_upsert(&app_state.app_config, target_name, XtreamCluster::Live, &updates).await + { error!("Failed to cascade Live updates to target {target_name}: {e}"); return; } @@ -1178,7 +1297,9 @@ impl InputWorker { let file_path = file_path.clone(); let query = match tokio::task::spawn_blocking(move || { BPlusTreeQuery::::try_new(&file_path) - }).await { + }) + .await + { Ok(Ok(query)) => Some(query), Ok(Err(err)) => { error!("Failed to open BPlusTreeQuery for {cluster}: {err}"); @@ -1191,10 +1312,7 @@ impl InputWorker { }; if let Some(query) = query { - entry.insert(DbHandle { - _guard: lock, - query: Arc::new(ParkingMutex::new(query)), - }); + entry.insert(DbHandle { _guard: lock, query: Arc::new(ParkingMutex::new(query)) }); } else { failed_clusters.insert(cluster); } @@ -1224,12 +1342,16 @@ impl InputWorker { if let ProviderIdType::Id(vid) = id { let stream_id = *vid; - if let Some(query) = Self::get_or_open_query(input_name, app_state, cluster, db_handles, failed_clusters).await { + if let Some(query) = + Self::get_or_open_query(input_name, app_state, cluster, db_handles, failed_clusters).await + { let query = Arc::clone(&query); let item = match tokio::task::spawn_blocking(move || { let mut guard = query.lock(); guard.query_zero_copy(&stream_id).ok().flatten() - }).await { + }) + .await + { Ok(item) => item, Err(err) => { error!("Failed to query item name for {stream_id}: {err}"); @@ -1245,6 +1367,7 @@ impl InputWorker { None } + #[allow(clippy::too_many_lines)] async fn process_task_static( input_name: &Arc, app_state_weak: Option<&Weak>, @@ -1253,9 +1376,8 @@ impl InputWorker { db_handles: &mut HashMap, failed_clusters: &mut HashSet, ) -> Result { - let app_state = app_state_weak - .and_then(Weak::upgrade) - .ok_or_else(|| shared::error::info_err!("AppState not available"))?; + let app_state = + app_state_weak.and_then(Weak::upgrade).ok_or_else(|| shared::error::info_err!("AppState not available"))?; let Some(input_base) = app_state.app_config.get_input_by_name(input_name) else { return Err(shared::error::info_err!("Input {} not found", input_name)); @@ -1265,8 +1387,15 @@ impl InputWorker { return Err(shared::error::info_err!("Input {} is disabled", input_name)); } - let needs_probe_connection = - Self::task_needs_provider_connection(task, input_base.input_type); + // Background metadata/probe tasks are low-priority. + // Never run them while a foreground playlist update is active. + if let Some(guard) = app_state.update_guard.try_playlist() { + drop(guard); + } else { + return Err(shared::error::info_err!("{}", TASK_ERR_UPDATE_IN_PROGRESS)); + } + + let needs_probe_connection = Self::task_needs_provider_connection(task, input_base.input_type); // Reserve provider capacity only for actual probe work (ffprobe paths). let provider_handle = if needs_probe_connection { @@ -1281,9 +1410,7 @@ impl InputWorker { let item_title = Self::get_item_name_static(input_name, &app_state, task, db_handles, failed_clusters).await; - let config_to_use = provider_handle - .as_ref() - .and_then(|handle| handle.allocation.get_provider_config()); + let config_to_use = provider_handle.as_ref().and_then(|handle| handle.allocation.get_provider_config()); let name_display = item_title.as_deref().map_or(String::new(), |n| format!(" \"{n}\"")); debug!("Processing task for {input_name}: {task}{name_display}"); @@ -1295,11 +1422,7 @@ impl InputWorker { // Determine input to use (may be alias) let input_to_use = config_to_use .filter(|alloc| alloc.name != input_base.name) - .and_then(|alloc| { - input_base.aliases.as_ref()? - .iter() - .find(|a| a.enabled && a.name == alloc.name) - }) + .and_then(|alloc| input_base.aliases.as_ref()?.iter().find(|a| a.enabled && a.name == alloc.name)) .map(|alias_def| { let mut temp_input = (*input_base).clone(); temp_input.url.clone_from(&alias_def.url); @@ -1331,10 +1454,32 @@ impl InputWorker { } } } else { - Self::execute_task_inner_static(&app_state, &client, &input_to_use, task, item_title.as_deref(), Some(handle), collector, db_handles, failed_clusters).await + Self::execute_task_inner_static( + &app_state, + &client, + &input_to_use, + task, + item_title.as_deref(), + Some(handle), + collector, + db_handles, + failed_clusters, + ) + .await } } else { - Self::execute_task_inner_static(&app_state, &client, &input_to_use, task, item_title.as_deref(), None, collector, db_handles, failed_clusters).await + Self::execute_task_inner_static( + &app_state, + &client, + &input_to_use, + task, + item_title.as_deref(), + None, + collector, + db_handles, + failed_clusters, + ) + .await }; if provider_handle.is_some() { @@ -1380,17 +1525,18 @@ impl InputWorker { let fetch_info = reason.contains(ResolveReason::Info); let will_probe = reason.contains(ResolveReason::Probe); - // If we are going to probe, release the cached handle to avoid holding a READ lock - // for along time (blocks writers) and also to avoid potential deadlocks if + // If we are going to probe, release the cached handle to avoid holding a READ lock + // for along time (blocks writers) and also to avoid potential deadlocks if // the probe function itself tries to acquire a WRITE lock later. if will_probe { db_handles.remove(&XtreamCluster::Video); } - let query_opt = if will_probe { - None + let query_opt = if will_probe { + None } else { - Self::get_or_open_query(&input.name, app_state, XtreamCluster::Video, db_handles, failed_clusters).await + Self::get_or_open_query(&input.name, app_state, XtreamCluster::Video, db_handles, failed_clusters) + .await }; match update_vod_metadata( @@ -1405,7 +1551,9 @@ impl InputWorker { fetch_info, will_probe, query_opt, - ).await { + ) + .await + { Ok(Some(props)) => { collector.add_vod(id.clone(), props); Ok(()) @@ -1426,7 +1574,8 @@ impl InputWorker { let query_opt = if will_probe { None } else { - Self::get_or_open_query(&input.name, app_state, XtreamCluster::Series, db_handles, failed_clusters).await + Self::get_or_open_query(&input.name, app_state, XtreamCluster::Series, db_handles, failed_clusters) + .await }; match update_series_metadata( @@ -1441,7 +1590,9 @@ impl InputWorker { fetch_info, will_probe, query_opt, - ).await { + ) + .await + { Ok(Some(props)) => { collector.add_series(id.clone(), props); Ok(()) @@ -1454,13 +1605,7 @@ impl InputWorker { // ProbeLive always probes, so we must never use a cached handle here. db_handles.remove(&XtreamCluster::Live); - match update_live_stream_metadata( - &app_state.app_config, - input, - id.clone(), - false, - None, - ).await { + match update_live_stream_metadata(&app_state.app_config, input, id.clone(), false, None).await { Ok(Some(props)) => { collector.add_live(id.clone(), props); Ok(()) @@ -1484,7 +1629,8 @@ impl InputWorker { *item_type, &app_state.active_provider, active_handle, - ).await + ) + .await } } } @@ -1553,15 +1699,10 @@ mod tests { assert_eq!(first_signal, TaskKey::Vod(42)); assert!(matches!( rx.try_recv(), - Err( - tokio::sync::mpsc::error::TryRecvError::Empty - | tokio::sync::mpsc::error::TryRecvError::Disconnected - ) + Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) )); - let entry = pending_tasks - .get(&TaskKey::Vod(42)) - .expect("pending entry should exist"); + let entry = pending_tasks.get(&TaskKey::Vod(42)).expect("pending entry should exist"); assert_eq!(entry.generation.load(Ordering::Relaxed), 1); let merged = entry.task.lock().await.clone(); @@ -1599,24 +1740,17 @@ mod tests { receiver: rx, pending_tasks: pending_tasks.clone(), app_state_weak: None, + update_pause_gate: Arc::new(RwLock::new(())), cancel_token: CancellationToken::new(), batch_buffer: BatchResultCollector::new(), db_handles: HashMap::new(), failed_clusters: HashSet::new(), }; - let requeued = worker - .finalize_processed_task_success(&key, 0, "input_a") - .await; + let requeued = worker.finalize_processed_task_success(&key, 0, "input_a").await; assert!(requeued); assert!(pending_tasks.contains_key(&key)); - assert_eq!( - worker - .receiver - .try_recv() - .expect("requeued signal should be present"), - key - ); + assert_eq!(worker.receiver.try_recv().expect("requeued signal should be present"), key); } #[tokio::test] @@ -1640,21 +1774,17 @@ mod tests { receiver: rx, pending_tasks: pending_tasks.clone(), app_state_weak: None, + update_pause_gate: Arc::new(RwLock::new(())), cancel_token: CancellationToken::new(), batch_buffer: BatchResultCollector::new(), db_handles: HashMap::new(), failed_clusters: HashSet::new(), }; - let requeued = worker - .finalize_processed_task_success(&key, 0, "input_b") - .await; + let requeued = worker.finalize_processed_task_success(&key, 0, "input_b").await; assert!(!requeued); assert!(!pending_tasks.contains_key(&key)); - assert!(matches!( - worker.receiver.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty) - )); + assert!(matches!(worker.receiver.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Empty))); } #[test] @@ -1668,10 +1798,7 @@ mod tests { delay: 0, }; - assert!(!InputWorker::task_needs_provider_connection( - &task, - InputType::Library - )); + assert!(!InputWorker::task_needs_provider_connection(&task, InputType::Library)); } #[test] @@ -1685,14 +1812,8 @@ mod tests { delay: 0, }; - assert!(InputWorker::task_needs_provider_connection( - &task, - InputType::M3u - )); - assert!(InputWorker::task_needs_provider_connection( - &task, - InputType::Xtream - )); + assert!(InputWorker::task_needs_provider_connection(&task, InputType::M3u)); + assert!(InputWorker::task_needs_provider_connection(&task, InputType::Xtream)); } #[test] @@ -1704,9 +1825,6 @@ mod tests { interval: 60, }; - assert!(InputWorker::task_needs_provider_connection( - &task, - InputType::Library - )); + assert!(InputWorker::task_needs_provider_connection(&task, InputType::Library)); } } diff --git a/backend/src/api/setup_api.rs b/backend/src/api/setup_api.rs index 112b8a584..4780cce15 100644 --- a/backend/src/api/setup_api.rs +++ b/backend/src/api/setup_api.rs @@ -2,8 +2,8 @@ use crate::api::api_utils::serve_file; use crate::auth::generate_password_from_input; use crate::model::validate_library_paths_from_dto; use crate::utils::{ - file_exists, get_default_path, get_default_web_root_path, read_api_proxy_file, read_config_file, read_sources_file, - sanitize_sources_for_persist, + file_exists, get_default_path, get_default_web_root_path, read_api_proxy_file, read_config_file, + read_sources_file, read_templates_file, resolve_template_persist_file_path, sanitize_sources_for_persist, }; use axum::extract::{Path, State}; use axum::http::StatusCode; @@ -15,10 +15,11 @@ use rand::Rng; use serde::{Deserialize, Serialize}; use serde_json::json; use shared::error::TuliproxError; +use shared::foundation::prepare_templates; use shared::info_err; use shared::model::{ ApiProxyConfigDto, ApiProxyServerInfoDto, AppConfigDto, ConfigApiDto, ConfigDto, ConfigPaths, SourcesConfigDto, - TokenResponse, WebAuthConfigDto, WebUiConfigDto, TOKEN_NO_AUTH, + PatternTemplate, TemplateDefinitionDto, TokenResponse, WebAuthConfigDto, WebUiConfigDto, TOKEN_NO_AUTH, }; use shared::utils::{default_kick_secs, hex_encode, DEFAULT_PORT, DEFAULT_WORKING_DIR, USER_FILE}; use std::collections::{HashMap, HashSet}; @@ -31,6 +32,7 @@ use tokio::sync::{oneshot, Mutex, RwLock}; use tower_http::services::ServeDir; const DEFAULT_SETUP_HOST: &str = "0.0.0.0"; +const DEFAULT_SETUP_CUSTOM_STREAM_RESPONSE_PATH: &str = "./resources"; const SETUP_REDACTED_SECRET_VALUE: &str = "__TULIPROX_SETUP_REDACTED__"; #[derive(Clone, Serialize, Deserialize)] @@ -171,6 +173,7 @@ fn create_default_config_dto() -> ConfigDto { web_root: get_default_web_root_path().display().to_string(), }, working_dir: get_default_path(DEFAULT_WORKING_DIR).display().to_string(), + custom_stream_response_path: Some(DEFAULT_SETUP_CUSTOM_STREAM_RESPONSE_PATH.to_string()), web_ui: Some(web_ui), ..ConfigDto::default() } @@ -181,6 +184,7 @@ fn create_default_draft() -> AppConfigDto { config: create_default_config_dto(), sources: SourcesConfigDto::default(), mappings: None, + templates: None, api_proxy: Some(ApiProxyConfigDto { server: vec![create_default_api_proxy_server()], user: vec![], @@ -202,7 +206,7 @@ fn build_initial_draft(paths: &ConfigPaths) -> AppConfigDto { if file_exists(&paths.sources_file_path) { match read_sources_file( paths.sources_file_path.as_str(), - true, + false, false, draft.config.get_hdhr_device_overview().as_ref(), ) { @@ -247,6 +251,13 @@ fn ensure_setup_defaults(config: &mut ConfigDto) { if config.working_dir.trim().is_empty() { config.working_dir = get_default_path(DEFAULT_WORKING_DIR).display().to_string(); } + if config + .custom_stream_response_path + .as_ref() + .is_none_or(|path| path.trim().is_empty()) + { + config.custom_stream_response_path = Some(DEFAULT_SETUP_CUSTOM_STREAM_RESPONSE_PATH.to_string()); + } if config.web_ui.is_none() { config.web_ui = Some(WebUiConfigDto::default()); @@ -421,6 +432,68 @@ fn has_unresolved_redacted_setup_values(app_config: &AppConfigDto) -> bool { }) } +fn setup_templates_from_request(app_config: &AppConfigDto) -> Option> { + app_config + .templates + .as_ref() + .and_then(|definition| (!definition.templates.is_empty()).then_some(definition.templates.clone())) +} + +fn setup_templates_to_persist(app_config: &AppConfigDto) -> Option { + if let Some(templates) = setup_templates_from_request(app_config) { + return Some(TemplateDefinitionDto { templates }); + } + app_config + .sources + .templates + .as_ref() + .filter(|templates| !templates.is_empty()) + .map(|templates| TemplateDefinitionDto { + templates: templates.clone(), + }) +} + +fn collect_setup_validation_templates( + app_config: &AppConfigDto, + template_file_path: &str, +) -> Result>, TuliproxError> { + let mut merged_templates: Vec = Vec::new(); + if let Some(templates) = setup_templates_from_request(app_config) { + merged_templates.extend(templates); + } + + if let Some((_, definition)) = read_templates_file(template_file_path, true)? { + merged_templates.extend(definition.templates); + } + if let Some(templates) = app_config.sources.templates.as_ref() { + merged_templates.extend(templates.clone()); + } + if let Some(templates) = app_config + .mappings + .as_ref() + .and_then(|mappings| mappings.mappings.templates.as_ref()) + { + merged_templates.extend(templates.clone()); + } + + if merged_templates.is_empty() { + Ok(None) + } else { + Ok(Some(merged_templates)) + } +} + +fn prepare_setup_validation_templates( + app_config: &AppConfigDto, + template_file_path: &str, +) -> Result>, TuliproxError> { + let Some(mut templates) = collect_setup_validation_templates(app_config, template_file_path)? else { + return Ok(None); + }; + prepare_templates(&mut templates)?; + Ok(Some(templates)) +} + async fn setup_healthcheck(State(state): State>) -> impl IntoResponse + Send { axum::Json(json!({ "status": "setup", @@ -633,6 +706,7 @@ async fn cleanup_temp_file(file_path: &FsPath) { enum SetupPersistAction<'a> { Config(&'a ConfigDto), Sources(&'a SourcesConfigDto), + Templates(&'a TemplateDefinitionDto), ApiProxy(&'a ApiProxyConfigDto), UserFile(&'a str), } @@ -642,6 +716,7 @@ impl SetupPersistAction<'_> { match self { Self::Config(payload) => persist_yaml_file(temp_path, *payload).await, Self::Sources(payload) => persist_yaml_file(temp_path, *payload).await, + Self::Templates(payload) => persist_yaml_file(temp_path, *payload).await, Self::ApiProxy(payload) => persist_yaml_file(temp_path, *payload).await, Self::UserFile(content) => tokio::fs::write(temp_path, content).await.map_err(|err| err.to_string()), } @@ -698,7 +773,27 @@ async fn setup_complete_inner( return (StatusCode::BAD_REQUEST, axum::Json(json!({ "error": err.to_string() }))).into_response(); } - if let Err(err) = req.app_config.sources.prepare(false, req.app_config.config.get_hdhr_device_overview().as_ref()) { + let template_file_path = PathBuf::from(resolve_template_persist_file_path( + req.app_config.config.template_path.as_deref(), + state.output_dir.to_string_lossy().as_ref(), + )); + let prepared_templates = + match prepare_setup_validation_templates(&req.app_config, template_file_path.to_string_lossy().as_ref()) { + Ok(templates) => templates, + Err(err) => { + return ( + StatusCode::BAD_REQUEST, + axum::Json(json!({ "error": err.to_string() })), + ) + .into_response(); + } + }; + + if let Err(err) = req.app_config.sources.prepare( + false, + req.app_config.config.get_hdhr_device_overview().as_ref(), + prepared_templates.as_deref(), + ) { return (StatusCode::BAD_REQUEST, axum::Json(json!({ "error": err.to_string() }))).into_response(); } @@ -715,12 +810,17 @@ async fn setup_complete_inner( Err(err) => return (StatusCode::BAD_REQUEST, axum::Json(json!({ "error": err }))).into_response(), }; - for file_path in [ + let template_definition_to_persist = setup_templates_to_persist(&req.app_config); + let mut persist_paths = vec![ &state.config_file_path, &state.source_file_path, &state.api_proxy_file_path, &state.user_file_path, - ] { + ]; + if template_definition_to_persist.is_some() { + persist_paths.push(&template_file_path); + } + for file_path in persist_paths { if let Err(err) = ensure_parent_dir(file_path).await { return ( StatusCode::INTERNAL_SERVER_ERROR, @@ -762,7 +862,7 @@ async fn setup_complete_inner( } let user_file_content = format!("{}\n", user_lines.join("\n")); - let pending_writes: Vec<(PathBuf, PathBuf, SetupPersistAction<'_>)> = vec![ + let mut pending_writes: Vec<(PathBuf, PathBuf, SetupPersistAction<'_>)> = vec![ ( state.config_file_path.clone(), create_setup_temp_path(&state.config_file_path), @@ -784,6 +884,13 @@ async fn setup_complete_inner( SetupPersistAction::UserFile(&user_file_content), ), ]; + if let Some(template_definition) = template_definition_to_persist.as_ref() { + pending_writes.push(( + template_file_path.clone(), + create_setup_temp_path(&template_file_path), + SetupPersistAction::Templates(template_definition), + )); + } for (index, (target_path, temp_path, action)) in pending_writes.iter().enumerate() { if let Err(err) = action.write(temp_path).await { diff --git a/backend/src/main.rs b/backend/src/main.rs index 1f401d0ef..f3457844a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -56,6 +56,10 @@ struct Args { #[arg(short = 'm', long = "mapping")] mapping_file: Option, + /// The template file + #[arg(short = 'T', long = "template")] + template_file: Option, + /// The target to process #[arg(short = 't', long)] target: Option>, @@ -201,12 +205,18 @@ fn print_info(app_config: &AppConfig) { info!("Source file: {:?}", &paths.sources_file_path); info!("Api Proxy File: {:?}", &paths.api_proxy_file_path); info!("Mapping path: {:?}", &paths.mapping_file_path.as_ref().map_or_else(|| "not used", |v| v.as_str())); + info!("Template path: {:?}", &paths.template_file_path.as_ref().map_or_else(|| "not used", |v| v.as_str())); if let Some(mapping_paths) = paths.mapping_files_used.as_ref() { for mapping_path in mapping_paths { info!("Mapping file loaded: {mapping_path}"); } } + if let Some(template_paths) = paths.template_files_used.as_ref() { + for template_path in template_paths { + info!("Template file loaded: {template_path}"); + } + } if let Some(cache) = config.reverse_proxy.as_ref().and_then(|r| r.cache.as_ref()) { if cache.enabled { @@ -250,6 +260,7 @@ fn get_file_paths(args: &Args) -> ConfigPaths { .map_or_else(|| utils::get_default_sources_file_path(&config_path), ToString::to_string), ); let mappings_file = args.mapping_file.as_ref().map(|p| resolve_env_var(p)); + let template_file = args.template_file.as_ref().map(|p| resolve_env_var(p)); ConfigPaths { config_path, @@ -257,6 +268,8 @@ fn get_file_paths(args: &Args) -> ConfigPaths { sources_file_path: sources_file, mapping_file_path: mappings_file, // need to be set after config read mapping_files_used: None, + template_file_path: template_file, // need to be set after config read + template_files_used: None, api_proxy_file_path: api_proxy_file, custom_stream_response_path: None, } diff --git a/backend/src/messaging.rs b/backend/src/messaging.rs index 342ebb015..9378143d6 100644 --- a/backend/src/messaging.rs +++ b/backend/src/messaging.rs @@ -301,6 +301,8 @@ mod tests { sources_file_path: String::new(), mapping_file_path: None, mapping_files_used: None, + template_file_path: None, + template_files_used: None, api_proxy_file_path: String::new(), custom_stream_response_path: None, })), diff --git a/backend/src/model/config/app.rs b/backend/src/model/config/app.rs index bac496d2d..55485f3b1 100644 --- a/backend/src/model/config/app.rs +++ b/backend/src/model/config/app.rs @@ -305,6 +305,8 @@ impl AppConfig { self.prepare_paths(); } else { self.prepare_mapping_path(); + self.prepare_template_path(); + self.prepare_custom_stream_response(); } self.prepare_sources()?; @@ -330,13 +332,29 @@ impl AppConfig { } } + fn set_template_path(&self, template_path: Option<&str>) { + let paths = self.paths.load_full(); + let mut new_paths = paths.as_ref().clone(); + let old_template_file_path = new_paths.template_file_path.as_deref(); + if old_template_file_path != template_path { + new_paths.template_file_path = template_path.map(ToString::to_string); + self.paths.store(Arc::new(new_paths)); + } + } + fn prepare_mapping_path(&self) { let config = self.config.load(); self.set_mapping_path(config.mapping_path.as_deref()); } + fn prepare_template_path(&self) { + let config = self.config.load(); + self.set_template_path(config.template_path.as_deref()); + } + fn prepare_paths(&self) { self.prepare_mapping_path(); + self.prepare_template_path(); self.prepare_custom_stream_response(); } diff --git a/backend/src/model/config/base.rs b/backend/src/model/config/base.rs index 69d45d4f8..1cd80897a 100644 --- a/backend/src/model/config/base.rs +++ b/backend/src/model/config/base.rs @@ -69,6 +69,7 @@ pub struct Config { pub backup_dir: Option, pub user_config_dir: Option, pub mapping_path: Option, + pub template_path: Option, pub custom_stream_response_path: Option, pub video: Option, pub schedules: Option>, @@ -190,6 +191,7 @@ impl From<&ConfigDto> for Config { backup_dir: dto.backup_dir.clone(), user_config_dir: dto.user_config_dir.clone(), mapping_path: dto.mapping_path.clone(), + template_path: dto.template_path.clone(), custom_stream_response_path: dto.custom_stream_response_path.clone(), video: dto.video.as_ref().map(Into::into), schedules: dto.schedules.as_ref().map(|s| s.iter().map(Into::into).collect()), diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index 5bf2b4ab4..d6e1cd67e 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -1,9 +1,9 @@ use crate::model::macros; -use shared::error::{TuliproxError, info_err_res}; +use shared::error::{info_err_res, TuliproxError}; use shared::model::{ConfigDto, LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; +use shared::utils::{default_metadata_path, Internable}; use std::path::PathBuf; use std::sync::Arc; -use shared::utils::{default_metadata_path, Internable}; #[derive(Debug, Clone, Default)] pub struct LibraryScanDirectory { @@ -44,7 +44,6 @@ pub struct LibraryPlaylistConfig { pub series_category: Arc, } - #[derive(Debug, Clone)] pub struct LibraryConfig { pub enabled: bool, @@ -62,11 +61,7 @@ impl LibraryConfig { } let scan_path = PathBuf::from(path); - let scan_path = if scan_path.is_relative() { - PathBuf::from(working_dir).join(scan_path) - } else { - scan_path - }; + let scan_path = if scan_path.is_relative() { PathBuf::from(working_dir).join(scan_path) } else { scan_path }; match scan_path.canonicalize() { Ok(path_buf) => Ok(path_buf.to_string_lossy().to_string()), @@ -89,11 +84,8 @@ impl LibraryConfig { // Resolve metadata path to absolute path based on working_dir let meta_path = PathBuf::from(&self.metadata.path); - let meta_path = if meta_path.is_relative() { - PathBuf::from(working_dir).join(&meta_path) - } else { - meta_path - }; + let meta_path = + if meta_path.is_relative() { PathBuf::from(working_dir).join(meta_path) } else { meta_path }; self.metadata.path = meta_path.to_string_lossy().to_string(); self.canonicalize_scan_directories(working_dir)?; @@ -112,7 +104,6 @@ macros::from_impl!(LibraryConfig); impl From<&LibraryConfigDto> for LibraryConfig { fn from(dto: &LibraryConfigDto) -> Self { - Self { enabled: dto.enabled, scan_directories: dto @@ -125,11 +116,7 @@ impl From<&LibraryConfigDto> for LibraryConfig { recursive: d.recursive, }) .collect(), - supported_extensions: dto - .supported_extensions - .iter() - .map(|ext| ext.to_lowercase()) - .collect(), + supported_extensions: dto.supported_extensions.iter().map(|ext| ext.to_lowercase()).collect(), metadata: LibraryMetadataConfig { path: dto.metadata.path.clone(), read_existing: LibraryMetadataReadConfig { @@ -177,31 +164,25 @@ mod tests { use std::time::{SystemTime, UNIX_EPOCH}; fn unique_missing_path() -> String { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time must be after unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!("tuliprox-library-missing-{nonce}")) - .to_string_lossy() - .to_string() + let nonce = + SystemTime::now().duration_since(UNIX_EPOCH).expect("system time must be after unix epoch").as_nanos(); + std::env::temp_dir().join(format!("tuliprox-library-missing-{nonce}")).to_string_lossy().to_string() } #[test] fn validate_library_paths_rejects_missing_scan_directory() { - let mut cfg = ConfigDto::default(); - cfg.working_dir = std::env::current_dir() - .expect("current_dir should be available") - .to_string_lossy() - .to_string(); - cfg.library = Some(LibraryConfigDto { - enabled: true, - scan_directories: vec![LibraryScanDirectoryDto { - path: unique_missing_path(), + let cfg = ConfigDto { + working_dir: std::env::current_dir() + .expect("current_dir should be available") + .to_string_lossy() + .to_string(), + library: Some(LibraryConfigDto { + enabled: true, + scan_directories: vec![LibraryScanDirectoryDto { path: unique_missing_path(), ..Default::default() }], ..Default::default() - }], + }), ..Default::default() - }); + }; let err = validate_library_paths_from_dto(&cfg).expect_err("missing scan directory must be rejected"); assert!(err.to_string().contains("Failed to canonicalize directory path")); @@ -209,40 +190,41 @@ mod tests { #[test] fn validate_library_paths_rejects_missing_scan_directory_even_when_disabled() { - let mut cfg = ConfigDto::default(); - cfg.working_dir = std::env::current_dir() - .expect("current_dir should be available") - .to_string_lossy() - .to_string(); - cfg.library = Some(LibraryConfigDto { - enabled: false, - scan_directories: vec![LibraryScanDirectoryDto { - path: unique_missing_path(), + let cfg = ConfigDto { + working_dir: std::env::current_dir() + .expect("current_dir should be available") + .to_string_lossy() + .to_string(), + library: Some(LibraryConfigDto { + enabled: false, + scan_directories: vec![LibraryScanDirectoryDto { path: unique_missing_path(), ..Default::default() }], ..Default::default() - }], + }), ..Default::default() - }); + }; - let err = - validate_library_paths_from_dto(&cfg).expect_err("missing scan directory must be rejected even when disabled"); + let err = validate_library_paths_from_dto(&cfg) + .expect_err("missing scan directory must be rejected even when disabled"); assert!(err.to_string().contains("Failed to canonicalize directory path")); } #[test] fn validate_library_paths_accepts_existing_scan_directory() { - let mut cfg = ConfigDto::default(); - cfg.working_dir = std::env::current_dir() - .expect("current_dir should be available") - .to_string_lossy() - .to_string(); - cfg.library = Some(LibraryConfigDto { - enabled: true, - scan_directories: vec![LibraryScanDirectoryDto { - path: std::env::temp_dir().to_string_lossy().to_string(), + let cfg = ConfigDto { + working_dir: std::env::current_dir() + .expect("current_dir should be available") + .to_string_lossy() + .to_string(), + library: Some(LibraryConfigDto { + enabled: true, + scan_directories: vec![LibraryScanDirectoryDto { + path: std::env::temp_dir().to_string_lossy().to_string(), + ..Default::default() + }], ..Default::default() - }], + }), ..Default::default() - }); + }; validate_library_paths_from_dto(&cfg).expect("existing scan directory should pass validation"); } diff --git a/backend/src/model/config/rename.rs b/backend/src/model/config/rename.rs index 1449a9470..f763cb6fa 100644 --- a/backend/src/model/config/rename.rs +++ b/backend/src/model/config/rename.rs @@ -1,22 +1,32 @@ -use std::sync::Arc; -use shared::model::{ConfigRenameDto, ItemField}; use crate::model::macros; +use shared::model::{ConfigRenameDto, ItemField}; +use std::sync::Arc; #[derive(Debug, Clone)] pub struct ConfigRename { pub field: ItemField, pub new_name: String, + pub raw_pattern: String, pub pattern: Arc, } macros::from_impl!(ConfigRename); impl From<&ConfigRenameDto> for ConfigRename { fn from(dto: &ConfigRenameDto) -> Self { - Self { - field: dto.field, - new_name: dto.new_name.clone(), - pattern: shared::model::REGEX_CACHE.get_or_compile(&dto.pattern).unwrap_or_else(|_| panic!("Invalid regex pattern {}", dto.pattern)), - } + let pattern_source = dto.t_pattern.as_ref().unwrap_or(&dto.pattern); + let pattern = shared::model::REGEX_CACHE + .get_or_compile(pattern_source) + .or_else(|err| { + log::warn!( + "Invalid rename regex pattern '{pattern_source}': {err}. Falling back to escaped literal pattern." + ); + shared::model::REGEX_CACHE.get_or_compile(®ex::escape(pattern_source)) + }) + .unwrap_or_else(|_| { + // Final fallback that avoids panicking on malformed user input. + shared::model::REGEX_CACHE.get_or_compile("$^").expect("hardcoded fallback regex '$^' must compile") + }); + Self { field: dto.field, new_name: dto.new_name.clone(), raw_pattern: dto.pattern.clone(), pattern } } } @@ -25,7 +35,46 @@ impl From<&ConfigRename> for ConfigRenameDto { Self { field: instance.field, new_name: instance.new_name.clone(), - pattern: instance.pattern.to_string() + pattern: if instance.raw_pattern.is_empty() { + instance.pattern.to_string() + } else { + instance.raw_pattern.clone() + }, + t_pattern: None, } } -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_dto_prefers_resolved_pattern() { + let dto = ConfigRenameDto { + field: ItemField::Name, + pattern: "!GROUP_PATTERN!".to_string(), + new_name: "Renamed".to_string(), + t_pattern: Some("^US.*$".to_string()), + }; + + let rename = ConfigRename::from(&dto); + assert_eq!(rename.pattern.as_str(), "^US.*$"); + assert_eq!(rename.raw_pattern, "!GROUP_PATTERN!"); + } + + #[test] + fn to_dto_preserves_original_placeholder_pattern() { + let dto = ConfigRenameDto { + field: ItemField::Name, + pattern: "!GROUP_PATTERN!".to_string(), + new_name: "Renamed".to_string(), + t_pattern: Some("^US.*$".to_string()), + }; + let rename = ConfigRename::from(&dto); + let mapped_back = ConfigRenameDto::from(&rename); + + assert_eq!(mapped_back.pattern, "!GROUP_PATTERN!"); + assert_eq!(mapped_back.t_pattern, None); + } +} diff --git a/backend/src/model/config/target.rs b/backend/src/model/config/target.rs index befe56aac..3945b8c40 100644 --- a/backend/src/model/config/target.rs +++ b/backend/src/model/config/target.rs @@ -3,13 +3,15 @@ use crate::model::config::trakt::TraktConfig; use crate::model::mapping::Mapping; use crate::model::{macros, ConfigRename, ConfigSort}; use arc_swap::ArcSwapOption; -use shared::model::{ConfigTargetDto, ConfigTargetOptions, HdHomeRunTargetOutputDto, M3uTargetOutputDto, - ProcessingOrder, StrmExportStyle, StrmTargetOutputDto, TargetOutputDto, TargetType, TraktConfigDto, XtreamTargetOutputDto}; -use shared::model::PlaylistItemType; -use std::sync::Arc; use shared::foundation::Filter; use shared::foundation::ValueProvider; +use shared::model::PlaylistItemType; +use shared::model::{ + ConfigTargetDto, ConfigTargetOptions, HdHomeRunTargetOutputDto, M3uTargetOutputDto, ProcessingOrder, + StrmExportStyle, StrmTargetOutputDto, TargetOutputDto, TargetType, TraktConfigDto, XtreamTargetOutputDto, +}; use shared::{apply_flags, create_bitset}; +use std::sync::Arc; create_bitset!(u8, XtreamTargetFlags, SkipLiveDirectSource, SkipVideoDirectSource, SkipSeriesDirectSource); create_bitset!(u8, StrmTargetFlags, Flat, UnderscoreWhitespace, Cleanup, AddQualityToFilename); @@ -50,11 +52,7 @@ impl From<&XtreamTargetOutputDto> for XtreamTargetOutput { (skip_series_direct_source, SkipSeriesDirectSource) ); - Self { - flags, - trakt: dto.trakt.as_ref().map(Into::into), - filter: dto.t_filter.clone(), - } + Self { flags, trakt: dto.trakt.as_ref().map(Into::into), filter: dto.t_filter.clone() } } } @@ -102,7 +100,6 @@ impl From<&M3uTargetOutput> for M3uTargetOutputDto { } } - #[derive(Debug, Clone)] pub struct StrmTargetOutput { pub directory: String, @@ -167,20 +164,12 @@ pub struct HdHomeRunTargetOutput { macros::from_impl!(HdHomeRunTargetOutput); impl From<&HdHomeRunTargetOutputDto> for HdHomeRunTargetOutput { fn from(dto: &HdHomeRunTargetOutputDto) -> Self { - Self { - device: dto.device.clone(), - username: dto.username.clone(), - use_output: dto.use_output, - } + Self { device: dto.device.clone(), username: dto.username.clone(), use_output: dto.use_output } } } impl From<&HdHomeRunTargetOutput> for HdHomeRunTargetOutputDto { fn from(instance: &HdHomeRunTargetOutput) -> Self { - Self { - device: instance.device.clone(), - username: instance.username.clone(), - use_output: instance.use_output, - } + Self { device: instance.device.clone(), username: instance.username.clone(), use_output: instance.use_output } } } @@ -255,7 +244,9 @@ impl ConfigTarget { } pub(crate) fn get_hdhomerun_output(&self) -> Option<&HdHomeRunTargetOutput> { - if let Some(TargetOutput::HdHomeRun(output)) = self.output.iter().find(|o| matches!(o, TargetOutput::HdHomeRun(_))) { + if let Some(TargetOutput::HdHomeRun(output)) = + self.output.iter().find(|o| matches!(o, TargetOutput::HdHomeRun(_))) + { Some(output) } else { None @@ -265,10 +256,26 @@ impl ConfigTarget { pub fn has_output(&self, tt: TargetType) -> bool { for target_output in &self.output { match target_output { - TargetOutput::Xtream(_) => { if tt == TargetType::Xtream { return true; } } - TargetOutput::M3u(_) => { if tt == TargetType::M3u { return true; } } - TargetOutput::Strm(_) => { if tt == TargetType::Strm { return true; } } - TargetOutput::HdHomeRun(_) => { if tt == TargetType::HdHomeRun { return true; } } + TargetOutput::Xtream(_) => { + if tt == TargetType::Xtream { + return true; + } + } + TargetOutput::M3u(_) => { + if tt == TargetType::M3u { + return true; + } + } + TargetOutput::Strm(_) => { + if tt == TargetType::Strm { + return true; + } + } + TargetOutput::HdHomeRun(_) => { + if tt == TargetType::HdHomeRun { + return true; + } + } } } false @@ -294,21 +301,30 @@ impl From<&ConfigTargetDto> for ConfigTarget { name: dto.name.clone(), options: dto.options.clone(), sort: dto.sort.as_ref().map(Into::into), - filter: dto.t_filter.as_ref().unwrap().clone(), + filter: dto.t_filter.clone().unwrap_or_default(), output: dto.output.iter().map(Into::into).collect(), rename: dto.rename.as_ref().map(|l| l.iter().map(Into::into).collect()), mapping_ids: dto.mapping.clone(), mapping: Arc::new(ArcSwapOption::new(None)), favourites: dto.favourites.as_ref().map(|f| f.iter().map(Into::into).collect()), processing_order: dto.processing_order, - watch: dto.watch.as_ref().map(|list| list.iter().filter_map(|s| - match shared::model::REGEX_CACHE.get_or_compile(s) { - Ok(re) => Some(re), - Err(e) => { - log::warn!("Invalid watch regex pattern '{s}': {e}"); - None - } - }).collect()), + watch: dto.watch.as_ref().and_then(|list| { + let compiled: Vec<_> = list + .iter() + .filter_map(|s| match shared::model::REGEX_CACHE.get_or_compile(s) { + Ok(re) => Some(re), + Err(e) => { + log::warn!("Invalid watch regex pattern '{s}': {e}"); + None + } + }) + .collect(); + if compiled.is_empty() { + None + } else { + Some(compiled) + } + }), use_memory_cache: dto.use_memory_cache, } } diff --git a/backend/src/processing/playlist_watch.rs b/backend/src/processing/playlist_watch.rs index dc77345f7..8f8269dc6 100644 --- a/backend/src/processing/playlist_watch.rs +++ b/backend/src/processing/playlist_watch.rs @@ -8,6 +8,9 @@ use crate::model::{AppConfig, MessageContent, WatchChanges}; use crate::utils; use crate::utils::{binary_deserialize, binary_serialize, file_exists_async}; +const WATCH_NOTIFICATION_LIST_LIMIT: usize = 120; +const WATCH_NOTIFICATION_SUMMARY_THRESHOLD: usize = 500; + pub async fn process_group_watch(app_config: &Arc, client: &reqwest::Client, target_name: &str, pl: &PlaylistGroup) { let mut new_tree = BTreeSet::new(); pl.channels.iter().for_each(|chan| { @@ -54,9 +57,40 @@ pub async fn process_group_watch(app_config: &Arc, client: &reqwest:: } async fn handle_watch_notification(app_config: &Arc, client: &reqwest::Client, added: &BTreeSet>, removed: &BTreeSet>, target_name: &str, group_name: &str) { - let added = added.iter().map(std::string::ToString::to_string).collect::>(); - let removed = removed.iter().map(std::string::ToString::to_string).collect::>(); + let added_count = added.len(); + let removed_count = removed.len(); + let total_changed = added_count.saturating_add(removed_count); + + let mut added = added.iter().map(std::string::ToString::to_string).collect::>(); + let mut removed = removed.iter().map(std::string::ToString::to_string).collect::>(); if !added.is_empty() || !removed.is_empty() { + if total_changed > WATCH_NOTIFICATION_SUMMARY_THRESHOLD { + added = if added_count > 0 { + vec![format!( + "{added_count} entries added. Detailed list suppressed for large update ({total_changed} total changes)." + )] + } else { + vec![] + }; + removed = if removed_count > 0 { + vec![format!( + "{removed_count} entries removed. Detailed list suppressed for large update ({total_changed} total changes)." + )] + } else { + vec![] + }; + } else { + if added.len() > WATCH_NOTIFICATION_LIST_LIMIT { + let omitted = added.len() - WATCH_NOTIFICATION_LIST_LIMIT; + added.truncate(WATCH_NOTIFICATION_LIST_LIMIT); + added.push(format!("... {omitted} more added entries omitted")); + } + if removed.len() > WATCH_NOTIFICATION_LIST_LIMIT { + let omitted = removed.len() - WATCH_NOTIFICATION_LIST_LIMIT; + removed.truncate(WATCH_NOTIFICATION_LIST_LIMIT); + removed.push(format!("... {omitted} more removed entries omitted")); + } + } let changes = WatchChanges { target: target_name.to_string(), @@ -81,4 +115,4 @@ async fn save_watch_tree(path: &Path, tree: &BTreeSet>) -> std::io::Res } let encoded: Vec = binary_serialize(&tree)?; tokio::fs::write(path, encoded).await -} \ No newline at end of file +} diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index c2770c562..d73f9d6de 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -34,7 +34,7 @@ use crate::repository::{load_input_playlist, persist_input_playlist, persist_pla use crate::repository::{CategoryKey, MemoryPlaylistSource, PlaylistSource}; use crate::utils::StepMeasure; use crate::utils::{debug_if_enabled, trace_if_enabled}; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use indexmap::IndexMap; use log::{debug, error, info, log_enabled, warn, Level}; use shared::concat_string; @@ -200,7 +200,7 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, V virtual_items: vec![], match_as_ascii: mapping.match_as_ascii, }; - script.eval(&mut accessor, templates); + script.eval(&mut accessor, templates.map(Vec::as_slice)); virtual_items.extend(accessor.virtual_items.into_iter().map(|(_, pli)| pli)); } } @@ -224,10 +224,10 @@ fn map_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Optio let mapping_binding = target.mapping.load(); let mappings = mapping_binding.as_ref()?; let valid_mappings = mappings.iter().filter(|m| m.mapper.as_ref().is_some_and(|v| !v.is_empty())); - let iter: Box> = Box::new(source.into_items()); + let iter: Box> = Box::new(source.into_items()); let mapped_iter = valid_mappings.fold(iter, |iter, mapping| { Box::new(iter.flat_map(move |chan| map_channel_and_flatten(chan, mapping))) - as Box> + as Box> }); let mut next_groups: IndexMap = IndexMap::new(); let mut grp_id: u32 = 0; @@ -515,7 +515,7 @@ async fn process_source( &mut errors, consume_input_source, ) - .await + .await { Ok(()) => { target_stats.push(TargetStats::success(&target.name)); @@ -560,11 +560,7 @@ async fn download_input( let need_download = !ctx.is_input_downloaded(&input.name).await; // Keep this lock for the whole critical section (download + persist/load + mark processed) // so parallel sources sharing the same input cannot observe a half-written state. - let _input_lock = if need_download { - Some(ctx.get_input_lock(&input.name).await) - } else { - None - }; + let _input_lock = if need_download { Some(ctx.get_input_lock(&input.name).await) } else { None }; let mut mark_as_processed = false; let playlist_download_result = if need_download { @@ -912,7 +908,7 @@ async fn process_playlist_for_target( target, ctx.playlist_state.as_ref(), ) - .await; + .await; step.stop("Persisting playlists"); log_memory_snapshot(format!("target '{}' after_persist", target.name).as_str()); result @@ -961,10 +957,10 @@ fn get_live_probe_interval_settings( return None; } target.get_xtream_output().map(|_| { - let (probe_delay, input_probe_live_interval_hours) = input_options.map_or( - (default_probe_delay_secs(), default_probe_live_interval()), - |options| (options.probe_delay, options.probe_live_interval_hours), - ); + let (probe_delay, input_probe_live_interval_hours) = input_options + .map_or((default_probe_delay_secs(), default_probe_live_interval()), |options| { + (options.probe_delay, options.probe_live_interval_hours) + }); (probe_delay, u64::from(input_probe_live_interval_hours) * 3600) }) } @@ -1195,8 +1191,8 @@ async fn process_watch( .filter(|pl| watches.iter().any(|r| r.is_match(&pl.title))) .map(|pl| process_group_watch(app_config, client, &target.name, pl)), ) - .for_each_concurrent(16, |f| f) - .await; + .for_each_concurrent(16, |f| f) + .await; true } else { @@ -1218,14 +1214,13 @@ pub async fn exec_processing( pre_processed_inputs: Option>>, acquired_permit: Option, ) { - let _guard = if let Some(permit) = acquired_permit { + let playlist_guard = if let Some(permit) = acquired_permit { Some(permit) } else if let Some(guard) = &update_guard { - let lock_result: Option = guard.try_playlist(); - if let Some(permit) = lock_result { + if let Some(permit) = guard.acquire_playlist_lock().await { Some(permit) } else { - warn!("Playlist update already in progress; update skipped."); + warn!("Playlist update lock is closed; update skipped."); if let Some(events) = event_manager.as_deref() { events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure)); } @@ -1235,6 +1230,13 @@ pub async fn exec_processing( None }; + // Pause background metadata/probe tasks for the full update lifecycle. + let _background_pause_guard = if let Some(manager) = metadata_manager.as_ref() { + Some(manager.acquire_update_pause_guard().await) + } else { + None + }; + log_memory_snapshot("exec_processing start"); // Initialize Context @@ -1253,8 +1255,20 @@ pub async fn exec_processing( }; let start_time = Instant::now(); - let (stats, errors) = process_sources(&ctx).await; + let process_result = std::panic::AssertUnwindSafe(process_sources(&ctx)).catch_unwind().await; + let Ok((stats, errors)) = process_result else { + error!("Playlist processing panicked"); + if let Some(events) = event_manager.as_deref() { + events.send_event(EventMessage::PlaylistUpdate(shared::model::PlaylistUpdateState::Failure)); + } + return; + }; log_memory_snapshot("exec_processing after_process_sources"); + + // Keep the update lock only for the critical processing section. + drop(playlist_guard); + debug!("Released playlist update lock; dispatching notifications and events"); + // log errors for err in &errors { error!("{}", err.message); diff --git a/backend/src/processing/processor/sort.rs b/backend/src/processing/processor/sort.rs index ca7fabbc4..d8ae90698 100644 --- a/backend/src/processing/processor/sort.rs +++ b/backend/src/processing/processor/sort.rs @@ -97,14 +97,8 @@ fn evaluate_sequence(plan: &SequencePlan, value: &str) -> SequenceMatch { fn compare_sequence_match(a: &SequenceMatch, b: &SequenceMatch, order: SortOrder) -> Ordering { match (a, b) { ( - SequenceMatch::Matched { - sequence_idx: idx_a, - captures: captures_a, - }, - SequenceMatch::Matched { - sequence_idx: idx_b, - captures: captures_b, - }, + SequenceMatch::Matched { sequence_idx: idx_a, captures: captures_a }, + SequenceMatch::Matched { sequence_idx: idx_b, captures: captures_b }, ) => { // Sequence index is an explicit priority list and is never reversed by DESC. let idx_ord = idx_a.cmp(idx_b); @@ -163,21 +157,13 @@ fn build_rule_cache_entry( match_as_ascii: bool, ) -> RuleCacheEntry { let Some(item) = item else { - return RuleCacheEntry { - filter_pass: false, - value: None, - sequence_match: None, - }; + return RuleCacheEntry { filter_pass: false, value: None, sequence_match: None }; }; let provider = ValueProvider { pli: item, match_as_ascii }; let filter_pass = rule.rule.filter.filter(&provider); if !filter_pass { - return RuleCacheEntry { - filter_pass: false, - value: None, - sequence_match: None, - }; + return RuleCacheEntry { filter_pass: false, value: None, sequence_match: None }; } let value = provider.get(rule.rule.field.as_str()); @@ -186,18 +172,11 @@ fn build_rule_cache_entry( _ => None, }; - RuleCacheEntry { - filter_pass: true, - value, - sequence_match, - } + RuleCacheEntry { filter_pass: true, value, sequence_match } } fn build_group_rule_cache(rule: &PreparedRule, groups: &[PlaylistGroup], match_as_ascii: bool) -> Vec { - groups - .iter() - .map(|group| build_rule_cache_entry(rule, group.channels.first(), match_as_ascii)) - .collect() + groups.iter().map(|group| build_rule_cache_entry(rule, group.channels.first(), match_as_ascii)).collect() } fn build_channel_rule_cache( @@ -205,23 +184,42 @@ fn build_channel_rule_cache( channels: &[shared::model::PlaylistItem], match_as_ascii: bool, ) -> Vec { - channels - .iter() - .map(|channel| build_rule_cache_entry(rule, Some(channel), match_as_ascii)) - .collect() + channels.iter().map(|channel| build_rule_cache_entry(rule, Some(channel), match_as_ascii)).collect() } fn reorder_by_indices(items: &mut Vec, indices: &[usize]) { - debug_assert_eq!(items.len(), indices.len()); + if items.len() != indices.len() { + log::error!( + "Invalid sort permutation: index length {} does not match item length {}", + indices.len(), + items.len() + ); + return; + } + + let mut seen = vec![false; items.len()]; + for &idx in indices { + if idx >= items.len() { + log::error!("Invalid sort permutation: index {idx} out of bounds for {} items", items.len()); + return; + } + if std::mem::replace(&mut seen[idx], true) { + log::error!("Invalid sort permutation: duplicate index {idx}"); + return; + } + } let mut original: Vec> = std::mem::take(items).into_iter().map(Some).collect(); items.reserve(indices.len()); for &idx in indices { - let item = original[idx] - .take() - .expect("index permutation must contain each source position exactly once"); - items.push(item); + if let Some(item) = original[idx].take() { + items.push(item); + } else { + log::error!("Invalid sort permutation: missing index {idx} after validation"); + items.extend(original.into_iter().flatten()); + return; + } } } @@ -304,11 +302,7 @@ pub(in crate::processing::processor) fn sort_playlist( true } -fn sort_groups( - groups: &mut Vec, - rules: &[ConfigSortRule], - match_as_ascii: bool, -) { +fn sort_groups(groups: &mut Vec, rules: &[ConfigSortRule], match_as_ascii: bool) { let group_rules: Vec<_> = rules .iter() .filter(|r| matches!(r.target, SortTarget::Group)) @@ -321,10 +315,8 @@ fn sort_groups( return; } - let rule_caches: Vec> = group_rules - .iter() - .map(|rule| build_group_rule_cache(rule, groups.as_slice(), match_as_ascii)) - .collect(); + let rule_caches: Vec> = + group_rules.iter().map(|rule| build_group_rule_cache(rule, groups.as_slice(), match_as_ascii)).collect(); let mut group_indices: Vec = (0..groups.len()).collect(); group_indices.sort_by(|left_idx, right_idx| { @@ -333,25 +325,15 @@ fn sort_groups( return ord; } - let order1 = groups[*left_idx] - .channels - .first() - .map_or(u32::MAX, |c| c.header.source_ordinal); - let order2 = groups[*right_idx] - .channels - .first() - .map_or(u32::MAX, |c| c.header.source_ordinal); + let order1 = groups[*left_idx].channels.first().map_or(u32::MAX, |c| c.header.source_ordinal); + let order2 = groups[*right_idx].channels.first().map_or(u32::MAX, |c| c.header.source_ordinal); order1.cmp(&order2) }); reorder_by_indices(groups, &group_indices); } -fn sort_channels_in_groups( - groups: &mut [PlaylistGroup], - rules: &[ConfigSortRule], - match_as_ascii: bool, -) { +fn sort_channels_in_groups(groups: &mut [PlaylistGroup], rules: &[ConfigSortRule], match_as_ascii: bool) { let channel_rules: Vec<_> = rules .iter() .filter(|r| matches!(r.target, SortTarget::Channel)) @@ -361,18 +343,14 @@ fn sort_channels_in_groups( if channel_rules.is_empty() { for group in groups { - group - .channels - .sort_by_key(|a| a.header.source_ordinal); + group.channels.sort_by_key(|a| a.header.source_ordinal); } return; } for group in groups { - let rule_caches: Vec> = channel_rules - .iter() - .map(|rule| build_channel_rule_cache(rule, &group.channels, match_as_ascii)) - .collect(); + let rule_caches: Vec> = + channel_rules.iter().map(|rule| build_channel_rule_cache(rule, &group.channels, match_as_ascii)).collect(); let mut channel_indices: Vec = (0..group.channels.len()).collect(); channel_indices.sort_by(|left_idx, right_idx| { @@ -381,10 +359,7 @@ fn sort_channels_in_groups( return ord; } - group.channels[*left_idx] - .header - .source_ordinal - .cmp(&group.channels[*right_idx].header.source_ordinal) + group.channels[*left_idx].header.source_ordinal.cmp(&group.channels[*right_idx].header.source_ordinal) }); reorder_by_indices(&mut group.channels, &channel_indices); @@ -398,17 +373,14 @@ mod tests { }; use crate::model::ConfigSortRule; use shared::foundation::Filter; - use shared::model::{ItemField, PlaylistGroup, PlaylistItem, PlaylistItemHeader, SortOrder, SortTarget, XtreamCluster}; + use shared::model::{ + ItemField, PlaylistGroup, PlaylistItem, PlaylistItemHeader, SortOrder, SortTarget, XtreamCluster, + }; use std::cmp::Ordering; use std::sync::Arc; fn make_group(id: u32, title: &str, channels: Vec) -> PlaylistGroup { - PlaylistGroup { - id, - title: title.into(), - channels, - xtream_cluster: XtreamCluster::Live, - } + PlaylistGroup { id, title: title.into(), channels, xtream_cluster: XtreamCluster::Live } } #[test] @@ -427,16 +399,16 @@ mod tests { ("T", "SD"), ("A", "FHD"), ] - .into_iter() - .enumerate() - .map(|(i, (name, quality))| PlaylistItem { - header: PlaylistItemHeader { - title: format!("Chanel {name} [{quality}]").into(), - source_ordinal: u32::try_from(i).unwrap(), - ..Default::default() - }, - }) - .collect(); + .into_iter() + .enumerate() + .map(|(i, (name, quality))| PlaylistItem { + header: PlaylistItemHeader { + title: format!("Chanel {name} [{quality}]").into(), + source_ordinal: u32::try_from(i).unwrap(), + ..Default::default() + }, + }) + .collect(); let channel_sort = ConfigSortRule { target: SortTarget::Channel, @@ -466,7 +438,10 @@ mod tests { "Chanel K [HD]", "Chanel Z [HD]", "Chanel T [SD]", - ].into_iter().map(Into::into).collect::>>(); + ] + .into_iter() + .map(Into::into) + .collect::>>(); let sorted = groups .into_iter() @@ -508,16 +483,16 @@ mod tests { "US| West e", "US| West f", ] - .into_iter() - .enumerate() - .map(|(i, name)| PlaylistItem { - header: PlaylistItemHeader { - title: name.to_string().into(), - source_ordinal: u32::try_from(i).unwrap(), - ..Default::default() - }, - }) - .collect(); + .into_iter() + .enumerate() + .map(|(i, name)| PlaylistItem { + header: PlaylistItemHeader { + title: name.to_string().into(), + source_ordinal: u32::try_from(i).unwrap(), + ..Default::default() + }, + }) + .collect(); let channel_sort = ConfigSortRule { target: SortTarget::Channel, @@ -564,7 +539,10 @@ mod tests { "US| East d", "US| West e", "US| West f", - ].into_iter().map(Into::into).collect::>>(); + ] + .into_iter() + .map(Into::into) + .collect::>>(); let sorted = groups .into_iter() @@ -598,12 +576,7 @@ mod tests { ]; channels.sort_by(|a, b| { - let ord = playlist_comparator( - Some(&sequence), - SortOrder::Desc, - &a.header.title, - &b.header.title, - ); + let ord = playlist_comparator(Some(&sequence), SortOrder::Desc, &a.header.title, &b.header.title); if ord == Ordering::Equal { a.header.source_ordinal.cmp(&b.header.source_ordinal) @@ -612,15 +585,9 @@ mod tests { } }); - let sorted = channels - .into_iter() - .map(|pli| pli.header.title) - .collect::>(); + let sorted = channels.into_iter().map(|pli| pli.header.title).collect::>(); - let expected = vec!["A-7", "A-1", "B-9", "B-2"] - .into_iter() - .map(Into::into) - .collect::>>(); + let expected = vec!["A-7", "A-1", "B-9", "B-2"].into_iter().map(Into::into).collect::>>(); assert_eq!(expected, sorted); } @@ -636,21 +603,9 @@ mod tests { }; let prepared = PreparedRule::new(&rule); - let pass_with_value = RuleCacheEntry { - filter_pass: true, - value: Some(Arc::from("A-2")), - sequence_match: None, - }; - let pass_without_value = RuleCacheEntry { - filter_pass: true, - value: None, - sequence_match: None, - }; - let fail_filter = RuleCacheEntry { - filter_pass: false, - value: None, - sequence_match: None, - }; + let pass_with_value = RuleCacheEntry { filter_pass: true, value: Some(Arc::from("A-2")), sequence_match: None }; + let pass_without_value = RuleCacheEntry { filter_pass: true, value: None, sequence_match: None }; + let fail_filter = RuleCacheEntry { filter_pass: false, value: None, sequence_match: None }; assert_eq!(compare_rule_entries(&prepared, &pass_with_value, &fail_filter), Ordering::Less); assert_eq!(compare_rule_entries(&prepared, &fail_filter, &pass_with_value), Ordering::Greater); @@ -690,16 +645,9 @@ mod tests { let mut groups = vec![make_group(1, "G1", channels)]; sort_channels_in_groups(groups.as_mut_slice(), &[first_rule, second_rule], false); - let sorted = groups[0] - .channels - .iter() - .map(|pli| pli.header.title.clone()) - .collect::>(); + let sorted = groups[0].channels.iter().map(|pli| pli.header.title.clone()).collect::>(); - let expected = vec!["A-2-x", "A-1-y"] - .into_iter() - .map(Into::into) - .collect::>>(); + let expected = vec!["A-2-x", "A-1-y"].into_iter().map(Into::into).collect::>>(); assert_eq!(expected, sorted); } @@ -709,22 +657,14 @@ mod tests { 1, "A", vec![PlaylistItem { - header: PlaylistItemHeader { - title: Arc::from("A-2-x"), - source_ordinal: 0, - ..Default::default() - }, + header: PlaylistItemHeader { title: Arc::from("A-2-x"), source_ordinal: 0, ..Default::default() }, }], ); let group_b = make_group( 2, "B", vec![PlaylistItem { - header: PlaylistItemHeader { - title: Arc::from("A-1-y"), - source_ordinal: 1, - ..Default::default() - }, + header: PlaylistItemHeader { title: Arc::from("A-1-y"), source_ordinal: 1, ..Default::default() }, }], ); @@ -740,14 +680,8 @@ mod tests { let mut groups = vec![group_a, group_b]; sort_groups(&mut groups, &[group_rule], false); - let sorted = groups - .iter() - .map(|group| group.channels[0].header.title.clone()) - .collect::>(); - let expected = vec!["A-1-y", "A-2-x"] - .into_iter() - .map(Into::into) - .collect::>>(); + let sorted = groups.iter().map(|group| group.channels[0].header.title.clone()).collect::>(); + let expected = vec!["A-1-y", "A-2-x"].into_iter().map(Into::into).collect::>>(); assert_eq!(expected, sorted); } } diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index c829900db..adfe3d88f 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -7,7 +7,9 @@ use crate::model::{ConfigInput, ConfigInputFlags, InputSource}; use crate::processing::parser::xtream::create_xtream_series_episode_url; use crate::processing::parser::xtream::parse_xtream_series_info; use crate::processing::processor::playlist::{PlaylistProcessingContext, ProcessingPipe}; -use crate::processing::processor::{create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags}; +use crate::processing::processor::{ + create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags, +}; use crate::ptt::ptt_parse_title; use crate::repository::persists_input_series_info; use crate::repository::{ @@ -16,12 +18,15 @@ use crate::repository::{ use crate::repository::{xtream_get_file_path, BPlusTreeQuery}; use crate::utils::{debug_if_enabled, xtream}; use log::{debug, error, info, log_enabled, trace, warn, Level}; +use parking_lot::Mutex; use serde_json::Value; use shared::error::TuliproxError; -use shared::model::{MediaQuality, PlaylistEntry, PlaylistItem, SeriesStreamProperties, StreamProperties, XtreamPlaylistItem, XtreamSeriesInfo}; +use shared::model::{ + MediaQuality, PlaylistEntry, PlaylistItem, SeriesStreamProperties, StreamProperties, XtreamPlaylistItem, + XtreamSeriesInfo, +}; use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster}; use std::collections::HashMap; -use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -50,15 +55,7 @@ pub async fn playlist_resolve_series( } provider_fpl.source.release_resources(XtreamCluster::Series); - playlist_resolve_series_info( - ctx, - errors, - processed_fpl, - resolve_options, - pipe, - target, - ) - .await; + playlist_resolve_series_info(ctx, errors, processed_fpl, resolve_options, pipe, target).await; if provider_fpl.is_memory() { sync_resolved_series_properties(provider_fpl, processed_fpl); @@ -68,16 +65,16 @@ pub async fn playlist_resolve_series( } #[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext, - _errors: &mut Vec, - fpl: &mut FetchedPlaylist<'_>, - resolve_options: ResolveOptions, - pipe: &ProcessingPipe, - target: &ConfigTarget) { +async fn playlist_resolve_series_info( + ctx: &PlaylistProcessingContext, + _errors: &mut Vec, + fpl: &mut FetchedPlaylist<'_>, + resolve_options: ResolveOptions, + pipe: &ProcessingPipe, + target: &ConfigTarget, +) { let filter = |pli: &PlaylistItem| { - if pli.header.xtream_cluster != XtreamCluster::Series - || pli.header.item_type != PlaylistItemType::SeriesInfo - { + if pli.header.xtream_cluster != XtreamCluster::Series || pli.header.item_type != PlaylistItemType::SeriesInfo { return false; } true @@ -91,24 +88,9 @@ async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext, let resolve_tmdb_enabled = fpl.input.has_flag(ConfigInputFlags::ResolveTmdb); let groups_to_add = if resolve_options.has_flag(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() { - queue_background_series_info( - ctx, - fpl, - filter, - &resolve_options, - skip_resolve, - resolve_tmdb_enabled, - ) + queue_background_series_info(ctx, fpl, filter, &resolve_options, skip_resolve, resolve_tmdb_enabled) } else { - process_immediate_series_info( - ctx, - fpl, - filter, - &resolve_options, - skip_resolve, - resolve_tmdb_enabled, - ) - .await + process_immediate_series_info(ctx, fpl, filter, &resolve_options, skip_resolve, resolve_tmdb_enabled).await }; // Apply pipe transformations to new groups @@ -128,16 +110,11 @@ async fn playlist_resolve_series_info(ctx: &PlaylistProcessingContext, } } -fn sync_resolved_series_properties( - provider_fpl: &mut FetchedPlaylist<'_>, - processed_fpl: &mut FetchedPlaylist<'_>, -) { +fn sync_resolved_series_properties(provider_fpl: &mut FetchedPlaylist<'_>, processed_fpl: &mut FetchedPlaylist<'_>) { let mut resolved_series_by_provider_id: HashMap = HashMap::new(); for pli in processed_fpl.items() { - if pli.header.xtream_cluster != XtreamCluster::Series - || pli.header.item_type != PlaylistItemType::SeriesInfo - { + if pli.header.xtream_cluster != XtreamCluster::Series || pli.header.item_type != PlaylistItemType::SeriesInfo { continue; } @@ -149,9 +126,7 @@ fn sync_resolved_series_properties( } if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() { - resolved_series_by_provider_id - .entry(provider_id) - .or_insert_with(|| properties.as_ref().clone()); + resolved_series_by_provider_id.entry(provider_id).or_insert_with(|| properties.as_ref().clone()); } } @@ -174,8 +149,7 @@ fn sync_resolved_series_properties( } if let Some(resolved) = resolved_series_by_provider_id.get(&provider_id) { - source_pli.header.additional_properties = - Some(StreamProperties::Series(Box::new(resolved.clone()))); + source_pli.header.additional_properties = Some(StreamProperties::Series(Box::new(resolved.clone()))); } } } @@ -279,7 +253,9 @@ async fn process_immediate_series_info( let guard = file_lock; let query = BPlusTreeQuery::::try_new(&xtream_path)?; Ok::<_, std::io::Error>((query, guard)) - }).await { + }) + .await + { Ok(Ok((query, guard))) => Some((query, guard)), Ok(Err(err)) => { error!("Failed to open BPlusTreeQuery for Series: {err}"); @@ -299,9 +275,20 @@ async fn process_immediate_series_info( // Pass the optional reference to the query let db_query_ref = db_query_holder.as_ref().map(Arc::clone); - match update_series_info_immediate(ctx, active_provider, input, pli, provider_id.clone(), &reasons, db_query_ref).await { + match update_series_info_immediate( + ctx, + active_provider, + input, + pli, + provider_id.clone(), + &reasons, + db_query_ref, + ) + .await + { Ok(Some(updated_props)) => { - pli.header.additional_properties = Some(StreamProperties::Series(Box::new(updated_props.clone()))); + pli.header.additional_properties = + Some(StreamProperties::Series(Box::new(updated_props.clone()))); batch.push((provider_id.clone(), updated_props)); if batch.len() >= BATCH_SIZE { @@ -309,7 +296,8 @@ async fn process_immediate_series_info( db_query_holder = None; _db_lock_holder = None; - let updates: Vec<(u32, SeriesStreamProperties)> = batch.iter() + let updates: Vec<(u32, SeriesStreamProperties)> = batch + .iter() .filter_map(|(id, props)| { if let ProviderIdType::Id(vid) = id { Some((*vid, props.clone())) @@ -328,7 +316,9 @@ async fn process_immediate_series_info( XtreamCluster::Series, &input.name, updates, - ).await { + ) + .await + { Ok(()) => batch.clear(), Err(err) => { error!( @@ -368,18 +358,16 @@ async fn process_immediate_series_info( // Release lock before final persist _db_lock_holder = None; - let updates: Vec<(u32, SeriesStreamProperties)> = batch.into_iter() + let updates: Vec<(u32, SeriesStreamProperties)> = batch + .into_iter() .filter_map(|(id, props)| if let ProviderIdType::Id(vid) = id { Some((vid, props)) } else { None }) .collect(); if !updates.is_empty() { - if let Err(err) = persist_input_series_info_batch( - &ctx.config, - &storage_path, - XtreamCluster::Series, - &input.name, - updates, - ).await { + if let Err(err) = + persist_input_series_info_batch(&ctx.config, &storage_path, XtreamCluster::Series, &input.name, updates) + .await + { error!("Failed to persist final batch series info: {err}"); } } @@ -400,7 +388,14 @@ fn expand_series_item(pli: &PlaylistItem, input: &ConfigInput) -> Option series_stream_props - .details - .as_ref() - .and_then(|d| d.episodes.as_ref()) - .is_some_and(|episodes| { + Some(StreamProperties::Series(series_stream_props)) => { + series_stream_props.details.as_ref().and_then(|d| d.episodes.as_ref()).is_some_and(|episodes| { episodes.iter().any(|ep| { let missing_video = !MediaQuality::is_valid_media_info(ep.video.as_deref()); let missing_audio = !MediaQuality::is_valid_media_info(ep.audio.as_deref()); missing_video || missing_audio }) - }), + }) + } _ => false, }; @@ -528,7 +524,8 @@ pub async fn update_series_metadata( db_query: Option>>>, ) -> Result, TuliproxError> { let working_dir = &app_config.config.load().working_dir; - let storage_path = get_input_storage_path(&input.name, working_dir).await + let storage_path = get_input_storage_path(&input.name, working_dir) + .await .map_err(|e| shared::error::info_err!("Storage path error: {e}"))?; if input.has_flag(ConfigInputFlags::XtreamSkipSeries) { @@ -652,7 +649,9 @@ pub async fn update_series_metadata( } } - let mut properties = props.unwrap(); + let mut properties = props.ok_or_else(|| { + shared::error::info_err!("No Series properties available after fallback creation for {display_id}") + })?; let resolve_tmdb_enabled = input.has_flag(ConfigInputFlags::ResolveTmdb); @@ -660,7 +659,8 @@ pub async fn update_series_metadata( if fetch_info && resolve_tmdb_enabled && (properties.tmdb.is_none() || properties.release_date.is_none()) - && !properties.name.is_empty() { + && !properties.name.is_empty() + { let config = app_config.config.load(); let library_config = config.library.as_ref(); let meta_resolver = MetadataResolver::new(library_config, client.clone()); @@ -689,12 +689,13 @@ pub async fn update_series_metadata( } // 3. API Name (fallback) - if (meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none()))) - && !properties.name.is_empty() { + if (meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none()))) && !properties.name.is_empty() { let title_already_tried = if let Some(t) = title_candidate { t == properties.name.as_ref() } else { false }; if !tried_title || !title_already_tried { debug!("Fallback to API Name '{}'...", properties.name); - meta = meta_resolver.resolve_from_title(&properties.name, properties.tmdb, false, resolve_tmdb_enabled).await; + meta = meta_resolver + .resolve_from_title(&properties.name, properties.tmdb, false, resolve_tmdb_enabled) + .await; } } @@ -746,11 +747,21 @@ pub async fn update_series_metadata( }; if active_handle.is_some() || temp_handle.is_some() { - let episode_url = create_xtream_series_episode_url(input_url, input_username, input_password, ep); + let episode_url = + create_xtream_series_episode_url(input_url, input_username, input_password, ep); // Specific logging for the user to follow - let missing_reason = if missing_video && missing_audio { "video/audio" } else if missing_video { "video" } else { "audio" }; - debug!("Probing Series Episode '{}' (S{}E{}) - Missing {}", ep.title, ep.season, ep.episode_num, missing_reason); + let missing_reason = if missing_video && missing_audio { + "video/audio" + } else if missing_video { + "video" + } else { + "audio" + }; + debug!( + "Probing Series Episode '{}' (S{}E{}) - Missing {}", + ep.title, ep.season, ep.episode_num, missing_reason + ); if let Some((_quality, raw_video, raw_audio)) = crate::utils::ffmpeg::probe_url( &episode_url, @@ -758,7 +769,9 @@ pub async fn update_series_metadata( analyze_duration, probe_size, ffprobe_timeout, - ).await { + ) + .await + { if let Some(v) = raw_video { ep.video = Some(v.to_string().into()); properties_updated = true; @@ -795,9 +808,16 @@ pub async fn update_series_metadata( if properties_updated || fetched_new { if save { if let Some(series_id) = series_id_opt { - persists_input_series_info(app_config, &storage_path, XtreamCluster::Series, &input.name, series_id, &properties) - .await - .map_err(|e| shared::error::info_err!("Persist error: {e}"))?; + persists_input_series_info( + app_config, + &storage_path, + XtreamCluster::Series, + &input.name, + series_id, + &properties, + ) + .await + .map_err(|e| shared::error::info_err!("Persist error: {e}"))?; debug_if_enabled!("Successfully updated Series metadata for ID {}", series_id); } diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 3d1191a5e..cb8ae5dda 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -1,24 +1,29 @@ use crate::api::model::{ActiveProviderManager, ProviderHandle}; -use crate::api::model::{ResolveReason, ResolveReasonSet, UpdateTask, ProviderIdType}; +use crate::api::model::{ProviderIdType, ResolveReason, ResolveReasonSet, UpdateTask}; use crate::library::MetadataResolver; -use crate::model::{ConfigInput, ConfigInputFlags}; use crate::model::FetchedPlaylist; use crate::model::InputSource; use crate::model::{AppConfig, ConfigTarget}; +use crate::model::{ConfigInput, ConfigInputFlags}; use crate::processing::processor::playlist::PlaylistProcessingContext; -use crate::processing::processor::{create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags}; +use crate::processing::processor::{ + create_resolve_options_function_for_xtream_target, ResolveOptions, ResolveOptionsFlags, +}; use crate::ptt::ptt_parse_title; use crate::repository::get_input_storage_path; use crate::repository::persist_input_vod_info; use crate::repository::persist_input_vod_info_batch; use crate::repository::{xtream_get_file_path, BPlusTreeQuery}; -use crate::utils::{trace_if_enabled, debug_if_enabled, xtream}; +use crate::utils::{debug_if_enabled, trace_if_enabled, xtream}; use log::{debug, error, info, log_enabled, trace, warn, Level}; +use parking_lot::Mutex; use serde_json::Value; use shared::error::TuliproxError; -use shared::model::{MediaQuality, PlaylistEntry, PlaylistItem, PlaylistItemType, StreamProperties, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo}; +use shared::model::{ + MediaQuality, PlaylistEntry, PlaylistItem, PlaylistItemType, StreamProperties, VideoStreamDetailProperties, + VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo, +}; use std::collections::HashMap; -use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -47,19 +52,14 @@ pub async fn playlist_resolve_vod( // Determine if we need to do anything if !resolve_options.has_flag(ResolveOptionsFlags::Resolve) && !do_probe - && !resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) { + && !resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) + { return; } provider_fpl.source.release_resources(XtreamCluster::Video); - playlist_resolve_vod_info( - ctx, - errors, - fpl, - resolve_options, - do_probe, - ).await; + playlist_resolve_vod_info(ctx, errors, fpl, resolve_options, do_probe).await; if provider_fpl.is_memory() { sync_resolved_vod_properties(provider_fpl, fpl); @@ -68,16 +68,11 @@ pub async fn playlist_resolve_vod( provider_fpl.source.obtain_resources().await; } -fn sync_resolved_vod_properties( - provider_fpl: &mut FetchedPlaylist<'_>, - processed_fpl: &mut FetchedPlaylist<'_>, -) { +fn sync_resolved_vod_properties(provider_fpl: &mut FetchedPlaylist<'_>, processed_fpl: &mut FetchedPlaylist<'_>) { let mut resolved_vod_by_provider_id: HashMap = HashMap::new(); for pli in processed_fpl.items() { - if pli.header.xtream_cluster != XtreamCluster::Video - || pli.header.item_type != PlaylistItemType::Video - { + if pli.header.xtream_cluster != XtreamCluster::Video || pli.header.item_type != PlaylistItemType::Video { continue; } @@ -89,9 +84,7 @@ fn sync_resolved_vod_properties( } if let Some(StreamProperties::Video(properties)) = pli.header.additional_properties.as_ref() { - resolved_vod_by_provider_id - .entry(provider_id) - .or_insert_with(|| properties.as_ref().clone()); + resolved_vod_by_provider_id.entry(provider_id).or_insert_with(|| properties.as_ref().clone()); } } @@ -114,8 +107,7 @@ fn sync_resolved_vod_properties( } if let Some(resolved) = resolved_vod_by_provider_id.get(&provider_id) { - source_pli.header.additional_properties = - Some(StreamProperties::Video(Box::new(resolved.clone()))); + source_pli.header.additional_properties = Some(StreamProperties::Video(Box::new(resolved.clone()))); } } } @@ -128,11 +120,8 @@ async fn playlist_resolve_vod_info( resolve_options: ResolveOptions, do_probe: bool, ) { - let filter = |pli: &PlaylistItem| { - if pli.header.xtream_cluster != XtreamCluster::Video - || pli.header.item_type != PlaylistItemType::Video - { + if pli.header.xtream_cluster != XtreamCluster::Video || pli.header.item_type != PlaylistItemType::Video { return false; } true @@ -141,33 +130,21 @@ async fn playlist_resolve_vod_info( let resolve_tmdb_enabled = fpl.input.has_flag(ConfigInputFlags::ResolveTmdb); if resolve_options.has_flag(ResolveOptionsFlags::Background) && ctx.metadata_manager.is_some() { - queue_background_vod_info( - ctx, - fpl, - filter, - &resolve_options, - do_probe, - resolve_tmdb_enabled, - ); + queue_background_vod_info(ctx, fpl, filter, &resolve_options, do_probe, resolve_tmdb_enabled); } else { - process_immediate_vod_info( - ctx, - fpl, - filter, - resolve_options, - do_probe, - resolve_tmdb_enabled, - ) - .await; + process_immediate_vod_info(ctx, fpl, filter, resolve_options, do_probe, resolve_tmdb_enabled).await; } } #[allow(clippy::too_many_lines)] -async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut FetchedPlaylist<'_>, - filter: impl Fn(&PlaylistItem) -> bool, - resolve_options: ResolveOptions, - do_probe: bool, - resolve_tmdb_enabled: bool, ) { +async fn process_immediate_vod_info( + ctx: &PlaylistProcessingContext, + fpl: &mut FetchedPlaylist<'_>, + filter: impl Fn(&PlaylistItem) -> bool, + resolve_options: ResolveOptions, + do_probe: bool, + resolve_tmdb_enabled: bool, +) { let input = fpl.input; let working_dir = &ctx.config.config.load().working_dir; @@ -209,7 +186,9 @@ async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut F let xtream_path = xtream_path.clone(); let query = match tokio::task::spawn_blocking(move || { BPlusTreeQuery::::try_new(&xtream_path) - }).await { + }) + .await + { Ok(Ok(query)) => Some(query), Ok(Err(err)) => { error!("Failed to open BPlusTreeQuery for VOD: {err}"); @@ -229,10 +208,13 @@ async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut F // Pass the optional reference to the query let db_query_ref = db_query_holder.as_ref().map(Arc::clone); - match update_vod_info_immediate(ctx, prov_mgr, input, pli, provider_id.clone(), &reasons, db_query_ref).await { + match update_vod_info_immediate(ctx, prov_mgr, input, pli, provider_id.clone(), &reasons, db_query_ref) + .await + { Ok(Some(updated_props)) => { // Update the current item in fpl - pli.header.additional_properties = Some(StreamProperties::Video(Box::new(updated_props.clone()))); + pli.header.additional_properties = + Some(StreamProperties::Video(Box::new(updated_props.clone()))); batch.push((provider_id.clone(), updated_props)); if batch.len() >= BATCH_SIZE { // Release lock before persisting to avoid deadlock (persist needs write lock) @@ -240,7 +222,8 @@ async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut F _db_lock_holder = None; // Filter for u32 IDs for persistence - let updates: Vec<(u32, VideoStreamProperties)> = batch.iter() + let updates: Vec<(u32, VideoStreamProperties)> = batch + .iter() .filter_map(|(id, props)| { if let ProviderIdType::Id(vid) = id { Some((*vid, props.clone())) @@ -259,7 +242,9 @@ async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut F XtreamCluster::Video, &input.name, updates, - ).await { + ) + .await + { Ok(()) => batch.clear(), Err(err) => { error!( @@ -298,12 +283,16 @@ async fn process_immediate_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut F // Release lock before final persist _db_lock_holder = None; - let updates: Vec<(u32, VideoStreamProperties)> = batch.into_iter() + let updates: Vec<(u32, VideoStreamProperties)> = batch + .into_iter() .filter_map(|(id, props)| if let ProviderIdType::Id(vid) = id { Some((vid, props)) } else { None }) .collect(); if !updates.is_empty() { - if let Err(err) = persist_input_vod_info_batch(&ctx.config, &storage_path, XtreamCluster::Video, &input.name, updates).await { + if let Err(err) = + persist_input_vod_info_batch(&ctx.config, &storage_path, XtreamCluster::Video, &input.name, updates) + .await + { error!("Failed to persist final batch VOD info: {err}"); } } @@ -342,9 +331,7 @@ fn check_needs_probe(pli: &PlaylistItem, reasons: &mut ResolveReasonSet) { let missing_audio = !MediaQuality::is_valid_media_info(details.and_then(|d| d.audio.as_deref())); missing_video || missing_audio } - None => { - true - } + None => true, _ => false, }; if needs_probe { @@ -368,25 +355,30 @@ fn check_resolve_tmdb( needs_info: bool, reasons: &mut ResolveReasonSet, ) { - if resolve_tmdb_enabled - && resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) - && needs_info { + if resolve_tmdb_enabled && resolve_options.has_flag(ResolveOptionsFlags::TmdbMissing) && needs_info { if let Some(StreamProperties::Video(video_stream_props)) = pli.header.additional_properties.as_ref() { let has_tmdb = video_stream_props.tmdb.is_some(); let has_date = video_stream_props.details.as_ref().and_then(|d| d.release_date.as_ref()).is_some(); if !has_tmdb || !has_date { - if !has_tmdb { reasons.set(ResolveReason::Tmdb); } - if !has_date { reasons.set(ResolveReason::Date); } + if !has_tmdb { + reasons.set(ResolveReason::Tmdb); + } + if !has_date { + reasons.set(ResolveReason::Date); + } } } } } -fn queue_background_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut FetchedPlaylist<'_>, - filter: impl Fn(&PlaylistItem) -> bool, - resolve_options: &ResolveOptions, - do_probe: bool, - resolve_tmdb_enabled: bool, ) { +fn queue_background_vod_info( + ctx: &PlaylistProcessingContext, + fpl: &mut FetchedPlaylist<'_>, + filter: impl Fn(&PlaylistItem) -> bool, + resolve_options: &ResolveOptions, + do_probe: bool, + resolve_tmdb_enabled: bool, +) { let Some(mgr) = ctx.metadata_manager.as_ref() else { return; }; @@ -406,11 +398,8 @@ fn queue_background_vod_info(ctx: &PlaylistProcessingContext, fpl: &mut FetchedP let reasons = check_resolve_reasons(resolve_options, do_probe, resolve_tmdb_enabled, pli); if !reasons.is_empty() { - let task = UpdateTask::ResolveVod { - id: provider_id, - reason: reasons, - delay: resolve_options.resolve_delay, - }; + let task = + UpdateTask::ResolveVod { id: provider_id, reason: reasons, delay: resolve_options.resolve_delay }; mgr.queue_task_background(input.name.clone(), task); } } @@ -439,10 +428,10 @@ async fn update_vod_info_immediate( fetch_info, reasons.contains(ResolveReason::Probe), db_query, - ).await + ) + .await } - /// Updates metadata for a single VOD item (Info + Probe). /// /// # Arguments @@ -465,7 +454,8 @@ pub async fn update_vod_metadata( db_query: Option>>>, ) -> Result, TuliproxError> { let working_dir = &app_config.config.load().working_dir; - let storage_path = get_input_storage_path(&input.name, working_dir).await + let storage_path = get_input_storage_path(&input.name, working_dir) + .await .map_err(|e| shared::error::info_err!("Storage path error: {e}"))?; // Check if we should skip based on input options @@ -603,8 +593,9 @@ pub async fn update_vod_metadata( } } - // Now it's safe to unwrap because we handled the None case above - let mut properties = props.unwrap(); + let Some(mut properties) = props else { + return Err(shared::error::info_err!("No VOD properties available after fallback creation for {display_id}")); + }; let resolve_tmdb_enabled = input.has_flag(ConfigInputFlags::ResolveTmdb); @@ -650,13 +641,16 @@ pub async fn update_vod_metadata( } // 3. API Name (fallback) - if (meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none()))) - && !properties.name.is_empty() { - let title_already_tried = if let Some(t) = title_candidate { t == properties.name.as_ref() } else { false }; + if (meta.is_none() || (meta.as_ref().is_some_and(|m| m.tmdb_id().is_none()))) && !properties.name.is_empty() + { + let title_already_tried = + if let Some(t) = title_candidate { t == properties.name.as_ref() } else { false }; if !tried_title || !title_already_tried { trace!("Fallback to API Name '{}'...", properties.name); - meta = meta_resolver.resolve_from_title(&properties.name, properties.tmdb, true, resolve_tmdb_enabled).await; + meta = meta_resolver + .resolve_from_title(&properties.name, properties.tmdb, true, resolve_tmdb_enabled) + .await; } } @@ -665,7 +659,8 @@ pub async fn update_vod_metadata( if let Some(o_name) = properties.details.as_ref().and_then(|d| d.o_name.as_deref()) { if !o_name.is_empty() && o_name != properties.name.as_ref() { trace!("Fallback to API Original Name '{o_name}'..."); - meta = meta_resolver.resolve_from_title(o_name, properties.tmdb, true, resolve_tmdb_enabled).await; + meta = + meta_resolver.resolve_from_title(o_name, properties.tmdb, true, resolve_tmdb_enabled).await; } } } @@ -697,18 +692,26 @@ pub async fn update_vod_metadata( properties.details = Some(VideoStreamDetailProperties::default()); } - let details = properties.details.as_ref().unwrap(); - let missing_video = !MediaQuality::is_valid_media_info(details.video.as_deref()); - let missing_audio = !MediaQuality::is_valid_media_info(details.audio.as_deref()); + let (missing_video, missing_audio) = match properties.details.as_ref() { + Some(details) => ( + !MediaQuality::is_valid_media_info(details.video.as_deref()), + !MediaQuality::is_valid_media_info(details.audio.as_deref()), + ), + None => (true, true), + }; if missing_video || missing_audio { let input_url = input.url.as_str(); let username = input.username.as_deref().unwrap_or(""); let password = input.password.as_deref().unwrap_or(""); let stream_url = crate::processing::parser::xtream::create_xtream_url( - XtreamCluster::Video, input_url, username, password, + XtreamCluster::Video, + input_url, + username, + password, &StreamProperties::Video(Box::new(properties.clone())), - true, true, + true, + true, ); let config = app_config.config.load(); @@ -732,7 +735,9 @@ pub async fn update_vod_metadata( analyze_duration, probe_size, ffprobe_timeout, - ).await { + ) + .await + { if let Some(details) = properties.details.as_mut() { if let Some(v) = raw_video { details.video = Some(v.to_string().into()); @@ -757,9 +762,16 @@ pub async fn update_vod_metadata( if properties_updated || fetched_new { if save { if let Some(stream_id) = stream_id_opt { - persist_input_vod_info(app_config, &storage_path, XtreamCluster::Video, &input.name, stream_id, &properties) - .await - .map_err(|e| shared::error::info_err!("Persist error: {e}"))?; + persist_input_vod_info( + app_config, + &storage_path, + XtreamCluster::Video, + &input.name, + stream_id, + &properties, + ) + .await + .map_err(|e| shared::error::info_err!("Persist error: {e}"))?; } debug_if_enabled!("Successfully updated VOD metadata for '{}' (ID: {})", display_title, display_id); diff --git a/backend/src/repository/user_repository.rs b/backend/src/repository/user_repository.rs index 663492694..e06abe2e3 100644 --- a/backend/src/repository/user_repository.rs +++ b/backend/src/repository/user_repository.rs @@ -516,6 +516,8 @@ mod tests { sources_file_path: String::new(), mapping_file_path: None, mapping_files_used: None, + template_file_path: None, + template_files_used: None, api_proxy_file_path: String::new(), custom_stream_response_path: None, }))), diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index 6abf89c1e..dbef5c89a 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -7,14 +7,22 @@ use crate::repository::{ use crate::utils; use crate::utils::{file_exists_async, file_reader}; use crate::utils::sys_utils::exit; -use crate::utils::{open_file, read_mappings_file, EnvResolvingReader, FileLockManager}; +use crate::utils::{ + open_file, read_mappings_file_unprepared, read_templates_file, EnvResolvingReader, + FileLockManager, +}; use arc_swap::{ArcSwap, ArcSwapAny}; use chrono::Local; use log::{error, info, warn}; use serde::Serialize; use shared::error::{info_err, info_err_res, TuliproxError}; -use shared::model::{ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, MsgKind, SourcesConfigDto, TargetUserDto}; -use shared::utils::CONSTANTS; +use shared::foundation::prepare_templates; +use shared::model::{ + ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, + MsgKind, PatternTemplate, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, +}; +use shared::utils::{CONSTANTS, TEMPLATE_FILE}; +use std::collections::{HashMap, HashSet}; use std::env; use std::fs::File; use std::io::{self, Read}; @@ -26,6 +34,12 @@ use shared::concat_string; use crate::utils::request::{is_uri}; use url::Url; +pub(crate) struct PreparedTemplateBundle { + pub(crate) definition: Option, + pub(crate) prepared: Option>, + pub(crate) files_used: Option>, +} + enum EitherReader { Left(L), Right(R), @@ -71,28 +85,16 @@ pub async fn read_api_proxy_config( } } -pub fn read_sources_file_from_path( +fn parse_sources_file_from_path( sources_file: &Path, resolve_env: bool, - include_computed: bool, - hdhr_config: Option<&HdHomeRunDeviceOverview>, ) -> Result { match open_file(sources_file) { Ok(file) => { let maybe_sources: Result = serde_saphyr::from_reader(config_file_reader(file, resolve_env)); match maybe_sources { - Ok(mut sources) => { - if resolve_env { - if let Err(err) = sources.prepare(include_computed, hdhr_config) { - return info_err_res!( - "Can't read the sources-config file: {}: {err}", - sources_file.display() - ); - } - } - Ok(sources) - } + Ok(sources) => Ok(sources), Err(err) => info_err_res!( "Can't read the sources-config file: {}: {err}", sources_file.display() @@ -106,17 +108,52 @@ pub fn read_sources_file_from_path( } } +pub fn read_sources_file_from_path_with_templates( + sources_file: &Path, + resolve_env: bool, + include_computed: bool, + hdhr_config: Option<&HdHomeRunDeviceOverview>, + prepared_templates: Option<&[shared::model::PatternTemplate]>, +) -> Result { + let mut sources = parse_sources_file_from_path(sources_file, resolve_env)?; + if resolve_env { + if let Err(err) = sources.prepare(include_computed, hdhr_config, prepared_templates) { + return info_err_res!( + "Can't read the sources-config file: {}: {err}", + sources_file.display() + ); + } + } + Ok(sources) +} + +pub fn read_sources_file_from_path( + sources_file: &Path, + resolve_env: bool, + include_computed: bool, + hdhr_config: Option<&HdHomeRunDeviceOverview>, +) -> Result { + read_sources_file_from_path_with_templates( + sources_file, + resolve_env, + include_computed, + hdhr_config, + None, + ) +} + pub fn read_sources_file( sources_file: &str, resolve_env: bool, include_computed: bool, hdhr_config: Option<&HdHomeRunDeviceOverview>, ) -> Result { - read_sources_file_from_path( + read_sources_file_from_path_with_templates( &PathBuf::from(sources_file), resolve_env, include_computed, hdhr_config, + None, ) } @@ -143,6 +180,132 @@ pub fn read_config_file( } } +#[allow(clippy::too_many_lines)] +pub(crate) fn read_templates( + template_file_path: Option<&str>, + resolve_env: bool, + sources_inline_templates: Option<&[PatternTemplate]>, + mappings_inline_templates: Option<&[PatternTemplate]>, +) -> Result { + struct TemplateNameUsage { + count: usize, + sources: HashSet, + } + + fn track_template_name( + usage: &mut HashMap, + template_name: &str, + source: &str, + ) { + let entry = usage + .entry(template_name.to_string()) + .or_insert_with(|| TemplateNameUsage { + count: 0, + sources: HashSet::new(), + }); + entry.count = entry.count.saturating_add(1); + entry.sources.insert(source.to_string()); + } + + let mut loaded_template_files: Vec = Vec::new(); + let mut merged_templates: Vec = Vec::new(); + let mut template_name_usage: HashMap = HashMap::new(); + + if let Some(path) = template_file_path { + if let Some((template_paths, definition)) = read_templates_file(path, resolve_env)? { + let file_source_label = if template_paths.is_empty() { + format!("template file: {path}") + } else if template_paths.len() == 1 { + format!("template file: {}", template_paths[0].display()) + } else { + format!( + "template files: {}", + template_paths + .iter() + .map(|template_path| template_path.display().to_string()) + .collect::>() + .join(", ") + ) + }; + for template in &definition.templates { + track_template_name( + &mut template_name_usage, + template.name.as_str(), + file_source_label.as_str(), + ); + } + merged_templates.extend(definition.templates); + loaded_template_files.extend(template_paths.iter().map(|p| p.display().to_string())); + } + } + + if let Some(templates) = sources_inline_templates { + for template in templates { + track_template_name( + &mut template_name_usage, + template.name.as_str(), + "source.yml inline templates", + ); + } + merged_templates.extend(templates.iter().cloned()); + } + if let Some(templates) = mappings_inline_templates { + for template in templates { + track_template_name( + &mut template_name_usage, + template.name.as_str(), + "mapping.yml inline templates", + ); + } + merged_templates.extend(templates.iter().cloned()); + } + + let mut duplicate_templates = template_name_usage + .iter() + .filter_map(|(name, usage)| { + if usage.count > 1 { + let mut sources = usage.sources.iter().cloned().collect::>(); + sources.sort(); + Some(format!("{name} [{}]", sources.join(", "))) + } else { + None + } + }) + .collect::>(); + if !duplicate_templates.is_empty() { + duplicate_templates.sort(); + return info_err_res!( + "Duplicate template names found across merged template sources: {}", + duplicate_templates.join("; ") + ); + } + + let files_used = if loaded_template_files.is_empty() { + None + } else { + Some(loaded_template_files) + }; + + if merged_templates.is_empty() { + return Ok(PreparedTemplateBundle { + definition: None, + prepared: None, + files_used, + }); + } + + let mut prepared_templates = merged_templates.clone(); + prepare_templates(&mut prepared_templates)?; + + Ok(PreparedTemplateBundle { + definition: Some(TemplateDefinitionDto { + templates: merged_templates, + }), + prepared: Some(prepared_templates), + files_used, + }) +} + pub fn read_app_config_dto( paths: &ConfigPaths, resolve_env: bool, @@ -153,30 +316,82 @@ pub fn read_app_config_dto( let api_proxy_file = paths.api_proxy_file_path.as_str(); let config = read_config_file(config_file, resolve_env, include_computed)?; - let sources = read_sources_file( - sources_file, - resolve_env, - include_computed, - config.get_hdhr_device_overview().as_ref(), - )?; - let mappings = if let Some(mappings_file) = paths.mapping_file_path.as_ref() { - read_mappings_file(mappings_file, resolve_env) - .unwrap_or(None) - .map(|(_, mappings)| mappings) + let mut sources = parse_sources_file_from_path(&PathBuf::from(sources_file), resolve_env)?; + let mut mappings = if let Some(mappings_file) = paths.mapping_file_path.as_ref() { + read_mappings_file_unprepared(mappings_file, resolve_env)?.map(|(_, mapping)| mapping) } else { None }; + let template_bundle = read_templates( + paths.template_file_path.as_deref().or(config.template_path.as_deref()), + resolve_env, + sources.templates.as_deref(), + mappings + .as_ref() + .and_then(|mapping| mapping.mappings.templates.as_deref()), + )?; + let templates = template_bundle.definition; + let prepared_templates = template_bundle.prepared; + + if resolve_env { + sources.prepare( + include_computed, + config.get_hdhr_device_overview().as_ref(), + prepared_templates.as_deref(), + )?; + if let Some(mapping) = mappings.as_mut() { + mapping + .mappings + .prepare(prepared_templates.as_deref())?; + } + } + + // Keep templates centralized in AppConfigDto.templates. + sources.templates = None; + if let Some(mapping) = mappings.as_mut() { + mapping.mappings.templates = None; + } + let api_proxy = read_api_proxy_file(api_proxy_file, resolve_env).unwrap_or(None); Ok(AppConfigDto { config, sources, mappings, + templates, api_proxy, }) } +fn apply_prepared_mappings( + app_config: &mut AppConfig, + paths: &mut ConfigPaths, + mappings_file: &str, + mapping_paths: Option<&Vec>, + mapping: &mut shared::model::MappingsDto, + prepared_templates: Option<&[PatternTemplate]>, +) { + match mapping.mappings.prepare(prepared_templates) { + Ok(()) => { + let mappings: crate::model::Mappings = crate::model::Mappings::from(&*mapping); + app_config.set_mappings(mappings_file, &mappings); + paths.mapping_files_used = mapping_paths.map(|items| { + items + .iter() + .map(|p| p.display().to_string()) + .collect::>() + }); + } + Err(err) => { + error!( + "Failed to prepare mapping '{mappings_file}': {err}. Skipping mapping registration." + ); + paths.mapping_files_used = None; + } + } +} + pub async fn prepare_sources_batch( sources: &mut SourcesConfigDto, include_computed: bool, @@ -276,20 +491,9 @@ pub async fn read_initial_app_config( let sources_file = paths.sources_file_path.as_str(); let config_dto = read_config_file(config_file, resolve_env, include_computed)?; - let mut sources_dto = read_sources_file( - sources_file, - resolve_env, - include_computed, - config_dto.get_hdhr_device_overview().as_ref(), - )?; - prepare_sources_batch(&mut sources_dto, include_computed).await?; - let sources: SourcesConfig = SourcesConfig::try_from(sources_dto)?; - let mut config: Config = Config::from(config_dto); - config.prepare(config_path).await?; - config.update_runtime(); if paths.mapping_file_path.is_none() { - let mut path = config.mapping_path.as_ref().map_or_else( + let mut path = config_dto.mapping_path.as_ref().map_or_else( || utils::get_default_mappings_path(config_path), ToString::to_string, ); @@ -298,6 +502,54 @@ pub async fn read_initial_app_config( } paths.mapping_file_path.replace(path); } + if paths.template_file_path.is_none() { + let mut path = config_dto.template_path.as_ref().map_or_else( + || utils::get_default_templates_path(config_path), + ToString::to_string, + ); + if resolve_env { + path = resolve_env_var(&path); + } + paths.template_file_path.replace(path); + } + + let mut sources_dto = parse_sources_file_from_path(&PathBuf::from(sources_file), resolve_env)?; + prepare_sources_batch(&mut sources_dto, include_computed).await?; + + let (mapping_paths, mut mappings_dto) = if let Some(mappings_file) = &paths.mapping_file_path { + match read_mappings_file_unprepared(mappings_file.as_str(), resolve_env) { + Ok(Some((mapping_paths, mappings))) => (Some(mapping_paths), Some(mappings)), + Ok(None) => (None, None), + Err(err) => return Err(err), + } + } else { + (None, None) + }; + + let template_bundle = read_templates( + paths.template_file_path.as_deref().or(config_dto.template_path.as_deref()), + resolve_env, + sources_dto.templates.as_deref(), + mappings_dto + .as_ref() + .and_then(|mapping| mapping.mappings.templates.as_deref()), + )?; + let prepared_templates = template_bundle.prepared; + let template_files_used = template_bundle.files_used; + + if resolve_env { + sources_dto.prepare( + include_computed, + config_dto.get_hdhr_device_overview().as_ref(), + prepared_templates.as_deref(), + )?; + } + sources_dto.templates = None; + + let sources: SourcesConfig = SourcesConfig::try_from(sources_dto)?; + let mut config: Config = Config::from(config_dto); + config.prepare(config_path).await?; + config.update_runtime(); let mut app_config = AppConfig { config: Arc::new(ArcSwap::from_pointee(config)), @@ -314,29 +566,24 @@ pub async fn read_initial_app_config( app_config.prepare(include_computed)?; //print_info(&app_config); - if let Some(mappings_file) = &paths.mapping_file_path { - match utils::read_mappings(mappings_file.as_str(), resolve_env) { - Ok(Some((mapping_paths, mappings))) => { - app_config.set_mappings(mappings_file, &mappings); - paths.mapping_files_used = { - let vec: Vec = mapping_paths - .iter() - .map(|p| p.display().to_string()) - .collect(); - - if vec.is_empty() { - None - } else { - Some(vec) - } - }; - app_config.paths.store(Arc::new(paths.clone())); - } - Ok(None) => info!("Mapping file: not used"), - Err(err) => exit!("{err}"), + if let Some(mappings_file) = paths.mapping_file_path.clone() { + if let Some(mapping) = mappings_dto.as_mut() { + apply_prepared_mappings( + &mut app_config, + paths, + mappings_file.as_str(), + mapping_paths.as_ref(), + mapping, + prepared_templates.as_deref(), + ); + } else { + info!("Mapping file: not used"); } } + paths.template_files_used = template_files_used; + app_config.paths.store(Arc::new(paths.clone())); + if server_mode { match read_api_proxy_config(&app_config, resolve_env).await { Ok(Some(api_proxy)) => app_config.set_api_proxy(api_proxy)?, @@ -499,6 +746,100 @@ where write_config_file(file_path, backup_dir, config, "source.yml").await } +pub async fn save_templates_config( + file_path: &str, + backup_dir: &str, + config: &TemplateDefinitionDto, +) -> Result<(), TuliproxError> { + write_config_file(file_path, backup_dir, config, TEMPLATE_FILE).await +} + +fn build_templates_to_persist( + app_state: &Arc, + dto: &SourcesConfigDto, +) -> Result, TuliproxError> { + let mut new_dto = dto.clone(); + let config = app_state.app_config.config.load(); + let paths = app_state.app_config.paths.load(); + + let existing_source_inline_templates = match read_sources_file( + paths.sources_file_path.as_str(), + true, + false, + None, + ) { + Ok(existing_sources) => existing_sources.templates, + Err(err) => { + warn!( + "Failed to read existing source.yml for template migration '{}': {err}", + paths.sources_file_path + ); + None + } + }; + + let mapping_inline_templates = if let Some(mapping_file_path) = paths.mapping_file_path.as_ref() { + read_mappings_file_unprepared(mapping_file_path, true)? + .map(|(_, mapping)| mapping) + .and_then(|mapping| mapping.mappings.templates) + } else { + None + }; + + let source_inline_templates = if new_dto.templates.is_some() { + new_dto.templates.as_deref() + } else { + existing_source_inline_templates.as_deref() + }; + + let template_file_path = paths.template_file_path.as_deref().or(config.template_path.as_deref()); + let template_bundle = read_templates( + template_file_path, + true, + source_inline_templates, + mapping_inline_templates.as_deref(), + )?; + let prepared_templates = template_bundle.prepared; + + new_dto.prepare( + true, + config.get_hdhr_device_overview().as_ref(), + prepared_templates.as_deref(), + )?; + + if let Some(templates) = new_dto.templates.clone() { + if templates.is_empty() { + Ok(None) + } else { + Ok(Some(TemplateDefinitionDto { templates })) + } + } else { + Ok(existing_source_inline_templates + .filter(|templates| !templates.is_empty()) + .map(|templates| TemplateDefinitionDto { templates })) + } +} + +pub fn validate_source_config_for_persist( + app_state: &Arc, + dto: &SourcesConfigDto, +) -> Result, TuliproxError> { + build_templates_to_persist(app_state, dto) +} + +pub async fn persist_templates_config( + app_state: &Arc, + template_definition: &TemplateDefinitionDto, +) -> Result<(), TuliproxError> { + let config = app_state.app_config.config.load(); + let paths = app_state.app_config.paths.load(); + let template_file = utils::resolve_template_persist_file_path( + paths.template_file_path.as_deref().or(config.template_path.as_deref()), + &paths.config_path, + ); + save_templates_config(&template_file, config.get_backup_dir().as_ref(), template_definition).await +} + pub async fn persist_source_config( app_state: &Arc, source_file_path: Option<&Path>, @@ -525,6 +866,7 @@ pub async fn persist_source_config( } pub async fn sanitize_sources_for_persist(mut source_config: SourcesConfigDto) -> SourcesConfigDto { + source_config.templates = None; for input in &mut source_config.inputs { if input .panel_api @@ -570,10 +912,10 @@ pub async fn validate_and_persist_source_config( app_state: &Arc, dto: SourcesConfigDto, ) -> Result { - { - let mut new_dto = dto.clone(); - let config = app_state.app_config.config.load(); - new_dto.prepare(true, config.get_hdhr_device_overview().as_ref())?; + let templates_to_persist = validate_source_config_for_persist(app_state, &dto)?; + + if let Some(template_definition) = templates_to_persist.as_ref() { + persist_templates_config(app_state, template_definition).await?; } persist_source_config(app_state, None, dto).await @@ -632,9 +974,12 @@ async fn persist_single_template(prefix: &str, kind: Option<&MsgKind>, template: // Treat existing file paths as file URLs if tokio::fs::metadata(template).await.is_ok() { - return Url::from_file_path(template) - .map(|u| u.to_string()) - .map_err(|()| info_err!("Failed to convert path to file URL: {template}")); + if let Ok(canonical_path) = tokio::fs::canonicalize(template).await { + if let Ok(url) = Url::from_file_path(&canonical_path) { + return Ok(url.to_string()); + } + } + return Ok(template.to_string()); } // It's a raw string, persist it @@ -664,9 +1009,14 @@ mod tests { use crate::utils::resolve_env_var; #[test] + #[allow(clippy::manual_unwrap_or_default)] fn test_resolve() { // Use PATH which exists on both Windows and Unix let resolved = resolve_env_var("${env:PATH}"); - assert_eq!(resolved, std::env::var("PATH").unwrap()); + let expected = match std::env::var("PATH") { + Ok(value) => value, + Err(_) => String::new(), + }; + assert_eq!(resolved, expected); } } diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index d38fac29f..3883dfc0e 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -2,7 +2,10 @@ use crate::utils::debug_if_enabled; use log::{debug, error, trace}; use path_clean::PathClean; use shared::error::str_to_io_error; -use shared::utils::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, DEFAULT_HOME_ENV_VAR, DEFAULT_WEB_DIR, DEFAULT_WEB_ROOT_ENV_VAR, MAPPING_FILE, SOURCE_FILE, USER_FILE}; +use shared::utils::{ + API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, DEFAULT_HOME_ENV_VAR, DEFAULT_WEB_DIR, DEFAULT_WEB_ROOT_ENV_VAR, + MAPPING_FILE, SOURCE_FILE, TEMPLATE_FILE, USER_FILE, +}; use std::borrow::Cow; use std::collections::HashSet; use std::fs::{File, OpenOptions}; @@ -108,6 +111,37 @@ pub fn get_default_mappings_path(config_path: &str) -> String { get_default_file_path(config_path, MAPPING_FILE) } +#[inline] +pub fn get_default_templates_path(config_path: &str) -> String { + get_default_file_path(config_path, TEMPLATE_FILE) +} + +#[inline] +pub fn resolve_template_persist_file_path(template_path: Option<&str>, config_path: &str) -> String { + let candidate = template_path + .filter(|path| !path.trim().is_empty()) + .map_or_else(|| get_default_templates_path(config_path), ToString::to_string); + let path = PathBuf::from(&candidate); + + if path.exists() { + if path.is_dir() { + return path.join(TEMPLATE_FILE).to_string_lossy().to_string(); + } + return path.to_string_lossy().to_string(); + } + + let looks_like_directory = candidate.ends_with('/') + || candidate.ends_with('\\') + || path.extension().is_none() + || path.extension().is_some_and(|ext| ext == "d"); + + if looks_like_directory { + path.join(TEMPLATE_FILE).to_string_lossy().to_string() + } else { + path.to_string_lossy().to_string() + } +} + #[inline] pub fn get_default_api_proxy_config_path(config_path: &str) -> String { get_default_file_path(config_path, API_PROXY_FILE) diff --git a/backend/src/utils/file/mapping_reader.rs b/backend/src/utils/file/mapping_reader.rs index 49980c216..7679b78e7 100644 --- a/backend/src/utils/file/mapping_reader.rs +++ b/backend/src/utils/file/mapping_reader.rs @@ -8,16 +8,11 @@ use std::path::{Path, PathBuf}; use shared::info_err; use shared::model::{MappingDefinitionDto, MappingDto, MappingsDto, PatternTemplate}; -fn read_mapping(mapping_file: &Path, resolve_var: bool, prepare_mappings: bool) -> Result, TuliproxError> { +fn read_mapping(mapping_file: &Path, resolve_var: bool) -> Result, TuliproxError> { if let Ok(file) = open_file(mapping_file) { let maybe_mapping: Result = serde_saphyr::from_reader(config_file_reader(file, resolve_var)); return match maybe_mapping { - Ok(mut mapping) => { - if prepare_mappings { - mapping.prepare()?; - } - Ok(Some(mapping)) - } + Ok(mapping) => Ok(Some(mapping)), Err(err) => { info_err_res!("{err}") } @@ -28,7 +23,7 @@ fn read_mapping(mapping_file: &Path, resolve_var: bool, prepare_mappings: bool) } fn read_mappings_from_file(mappings_file: &Path, resolve_env: bool) -> Result, MappingsDto)>, TuliproxError> { - match read_mapping(mappings_file, resolve_env, true) { + match read_mapping(mappings_file, resolve_env) { Ok(mappings) => { match mappings { None => Ok(None), @@ -60,7 +55,7 @@ fn merge_mappings(mappings: Vec) -> Vec { map.into_values().collect() } -fn merge_mapping_definitions(mappings: Vec) -> Result, TuliproxError> { +fn merge_mapping_definitions(mappings: Vec) -> MappingsDto { let mut merged_templates: Vec = Vec::new(); let mut merged_mapping: Vec = Vec::new(); @@ -72,14 +67,12 @@ fn merge_mapping_definitions(mappings: Vec) -> Result Result, MappingsDto)>, TuliproxError> { @@ -99,7 +92,7 @@ fn read_mappings_from_directory(path: &Path, resolve_env: bool) -> Result