diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddad6739..c56387942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - **Panel API Integration**: Optional integration to renew expired accounts or provision new aliases when connections are exhausted. - **Playlist Caching**: Added `cache_duration` to inputs, allowing configurable provider playlist cache times during subsequent updates (e.g., `60s`, `5m` `12h`, `1d`). - **Database Viewer**: New CLI flags `--dbx` and `--dbm` to inspect internal database content. +- **Added `disk_based_processing`** (boolean, default `false`) to `config.yml`. When enabled, input playlists are processed from disk instead of memory. ## ⚙️ Engine & Storage Optimizations - **Slotted Page Architecture**: Improved space utilization and support for variable-length keys. @@ -30,6 +31,8 @@ - **Atomic I/O Layer**: Refactored for atomic writes and file locking, ensuring data integrity. - **B+Tree Compaction**: Reclaim space after deletions or mass updates. - **Batch Upsert**: Significantly higher throughput during mass inserts/updates. +- **Disk-Based Provider Processing**: New `disk_based_processing` config option massively reduces RAM usage by streaming playlist data from disk (BPlusTree) during updates. +- **String Interning**: Implemented `Arc` string interning for playlist items to further reduce memory footprint. ## 🔍 Mapping & Filtering Enhancements - **Accent-Independent Matching**: Integrated `match_as_ascii` flag for robust text matching (e.g., "Cinema" matches "Cinéma"). diff --git a/Cargo.lock b/Cargo.lock index dc33848ee..29572b2bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1156,7 +1156,7 @@ dependencies = [ [[package]] name = "frontend" -version = "3.2.37" +version = "3.2.38" dependencies = [ "anyhow", "base64", @@ -1168,9 +1168,9 @@ dependencies = [ "futures-signals", "gloo-render 0.2.0", "gloo-storage 0.3.0", - "gloo-timers 0.2.6", + "gloo-timers 0.3.0", "gloo-utils 0.2.0", - "implicit-clone", + "implicit-clone 0.6.0", "js-sys", "log", "prost", @@ -1691,8 +1691,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" dependencies = [ - "futures-channel", - "futures-core", "js-sys", "wasm-bindgen", ] @@ -2252,6 +2250,15 @@ dependencies = [ "indexmap", ] +[[package]] +name = "implicit-clone" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1689b939ee35e3a075b0834b5672efd43aec8a6e81a1c6002b76a5ca2f211ae0" +dependencies = [ + "implicit-clone-derive", +] + [[package]] name = "implicit-clone-derive" version = "0.1.2" @@ -2505,15 +2512,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lz4_flex" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" -dependencies = [ - "twox-hash", -] - [[package]] name = "lz4_flex" version = "0.12.0" @@ -3863,9 +3861,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "0.0.12" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef7e021087b4b525edcbb93045ca3f1dafeff7800b3c7960a90a75f197e257da" +checksum = "be7f28bb35aab6f9cdc5f464f9bbda0b1e3908766e819557111cc13e58ce7915" dependencies = [ "ahash", "annotate-snippets", @@ -3992,7 +3990,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.2.37" +version = "3.2.38" dependencies = [ "base64", "bitflags 2.10.0", @@ -4008,7 +4006,7 @@ dependencies = [ "indexmap", "js-sys", "log", - "lz4_flex 0.12.0", + "lz4_flex", "path-clean", "pest", "pest_derive", @@ -4566,7 +4564,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.2.37" +version = "3.2.38" dependencies = [ "arc-swap", "async-compression", @@ -4597,7 +4595,7 @@ dependencies = [ "libc", "log", "lol_html", - "lz4_flex 0.11.5", + "lz4_flex", "mime", "mime_guess", "notify", @@ -4697,9 +4695,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -5351,7 +5349,7 @@ dependencies = [ "console_error_panic_hook", "futures", "gloo 0.10.0", - "implicit-clone", + "implicit-clone 0.4.9", "indexmap", "js-sys", "prokio", diff --git a/README.md b/README.md index baed2387d..d0694072f 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Top level entries in the config files are: * `config_hot_reload` _optional_, default false. * `sleep_timer_mins` _optional_, used for closing stream after the given minutes. * `accept_unsecure_ssl_certificates` _optional_, default false. +* `disk_based_processing` _optional_, default false. When set to true, input playlists are processed from disk to save RAM. * `library` _optional_, for local media ### 1.1. `process_parallel` diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2d1545c3b..94757ca35 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "tuliprox" -version = "3.2.37" +version = "3.2.38" edition = "2021" -rust-version = "1.87.0" +rust-version = "1.89.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,13 +11,13 @@ lol_html = "2.7" handlebars = "6" shared = { version = "3", path = "../shared" } serde = { version = "1.0", features = ["derive", "rc"] } -serde-saphyr = "0.0.12" +serde-saphyr = "0.0.13" serde_json = { version = "1", features = ["raw_value"] } quick-xml = { version = "0.38", features = ["async-tokio"] } regex = "1.12" clap = { version = "4", features = ["derive"] } reqwest = { version = "0", features = ["blocking", "json", "stream", "rustls-tls", "socks"] } -url = "2.5.7" +url = "2.5.8" chrono = "0.4" cron = "0.15" axum = { version = "0.8" , features = ["macros", "default", "ws"]} @@ -44,7 +44,7 @@ flate2 = "1.1" blake3 = "1.8" bytes = "1.11" tokio-stream = { version = "0.1", features = ["sync"] } -tokio = { version = "1.48", features = ["rt-multi-thread", "parking_lot", "fs"] } +tokio = { version = "1.49", features = ["rt-multi-thread", "parking_lot", "fs"] } #tokio = { version = "1.46", features = ["rt-multi-thread", "parking_lot", "fs", "tracing"] } #console-subscriber = "0" #tracing = "0.1" @@ -58,7 +58,7 @@ rayon = "1.11" base64 = "0.22" paste = "1.0" indexmap = "2" -lz4_flex = "0.11" +lz4_flex = "0.12" fs2 = "0.4" notify = "8.2" arc-swap = "1.8" diff --git a/backend/src/api/config_file.rs b/backend/src/api/config_file.rs index 6c25730eb..533d02661 100644 --- a/backend/src/api/config_file.rs +++ b/backend/src/api/config_file.rs @@ -10,6 +10,7 @@ use shared::model::{ConfigPaths, ConfigType}; use std::path::Path; use std::sync::Arc; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ConfigFile { Config, ApiProxy, diff --git a/backend/src/api/config_watch.rs b/backend/src/api/config_watch.rs index eb829c764..2de1419c0 100644 --- a/backend/src/api/config_watch.rs +++ b/backend/src/api/config_watch.rs @@ -39,10 +39,6 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke } }); - // Use recommended_watcher() to automatically select the best implementation - // for your platform. The `EventHandler` passed to this constructor can be a - // closure, a `std::sync::mpsc::Sender`, a `crossbeam_channel::Sender`, or - // another type the trait is implemented for. let mut watcher = recommended_watcher(std_tx).map_err(|err| TuliproxError::new(TuliproxErrorKind::Info, format!("Failed to init config file watcher {err}")))?; watcher.watch(path, recursive_mode).map_err(|err| TuliproxError::new(TuliproxErrorKind::Info, format!("Failed to start config file watcher {err}")))?; info!("Watching config file changes {}", path.display()); @@ -62,6 +58,10 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke let _keep_watcher_alive = watcher; + let mut debounce_timer = Box::pin(tokio::time::sleep(tokio::time::Duration::from_millis(0))); + let mut timer_active = false; + let mut pending_configs: HashMap = HashMap::new(); + loop { tokio::select! { biased; @@ -75,27 +75,49 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke match res { Ok(event) => { if let EventKind::Access(AccessKind::Close(AccessMode::Write)) = event.kind { - for path in event.paths { - if let Some((config_file, _is_dir)) = files.get(&path) { - if let Err(err) = config_file.reload(&path, &watcher_app_state).await { - handle_error(err, &path); - } - } else if recursive_mode == RecursiveMode::Recursive && path.extension().is_some_and(|ext| ext == "yml") { - for (key, (config_file, is_dir)) in &files { - if *is_dir && path.starts_with(key) { - if let Err(err) = config_file.reload(&path, &watcher_app_state).await { - handle_error(err, &path); + let mut trigger = false; + for path in event.paths { + let mut resolved = None; + if let Some((config_file, _is_dir)) = files.get(&path) { + resolved = Some(*config_file); + } else if recursive_mode == RecursiveMode::Recursive && path.extension().is_some_and(|ext| ext == "yml") { + for (key, (config_file, is_dir)) in &files { + if *is_dir && path.starts_with(key) { + resolved = Some(*config_file); + break; } } } + + if let Some(config_file) = resolved { + // Consolidate updates + let effective = if config_file == ConfigFile::SourceFile { ConfigFile::Sources } else { config_file }; + pending_configs.insert(effective, path); + trigger = true; + } + } + + if trigger { + debounce_timer = Box::pin(tokio::time::sleep(tokio::time::Duration::from_millis(500))); + timer_active = true; } } } - } Err(e) => error!("watch error: {e:?}"), } } + () = &mut debounce_timer, if timer_active => { + timer_active = false; + if !pending_configs.is_empty() { + for (config_file, path) in pending_configs.drain() { + if let Err(err) = config_file.reload(&path, &watcher_app_state).await { + handle_error(err, &path); + } + } + } + } + else => break, // Channel closed } } diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index e4f8e861d..d594411d1 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -14,12 +14,12 @@ use crate::api::api_utils::{json_or_bin_response}; fn group_playlist_items( cluster: XtreamCluster, iter: impl Iterator, - get_group: fn(&T) -> String, + get_group: fn(&T) -> Arc, ) -> Vec where T: Serialize + Into, { - let mut groups: IndexMap> = IndexMap::new(); + let mut groups: IndexMap, Vec> = IndexMap::new(); for item in iter { let group_key = get_group(&item); @@ -115,7 +115,7 @@ async fn grouped_channels( )) } -pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&str>) -> impl axum::response::IntoResponse + Send { +pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&str>) -> impl IntoResponse + Send { if let Some(target) = cfg_target { if target.has_output(TargetType::Xtream) { let live_channels = grouped_channels(cfg, target, XtreamCluster::Live).await; @@ -150,7 +150,10 @@ pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cf let (result, errors) = match input.input_type { InputType::M3u | InputType::M3uBatch => m3u::download_m3u_playlist(client, cfg, input).await, - InputType::Xtream | InputType::XtreamBatch => xtream::download_xtream_playlist(cfg, client, input, None).await, + InputType::Xtream | InputType::XtreamBatch => { + let (pl, err, _) = xtream::download_xtream_playlist(cfg, client, input, None).await; + (pl, err) + } InputType::Library => { return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({ "error": "Library inputs are not supported on this endpoint"}))).into_response(); } diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 05a0e0030..7e5de296d 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -21,7 +21,7 @@ use axum::response::IntoResponse; use log::{debug, error}; use serde::Deserialize; use shared::model::{PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, XtreamCluster}; -use shared::utils::{is_hls_url, replace_url_extension, sanitize_sensitive_info, CUSTOM_VIDEO_PREFIX, HLS_EXT}; +use shared::utils::{intern, is_hls_url, replace_url_extension, sanitize_sensitive_info, CUSTOM_VIDEO_PREFIX, HLS_EXT}; use std::sync::Arc; const PLAYLIST_TEMPLATE: &str = r"#EXTM3U @@ -132,6 +132,7 @@ pub(in crate::api) async fn handle_hls_stream_request( &input_source, Some(&headers), None, + false ) .await { @@ -196,7 +197,7 @@ async fn resolve_stream_channel( provider_id: 0, item_type: PlaylistItemType::LiveHls, cluster: XtreamCluster::Live, - group: "Unknown".to_string(), + group: intern("Unknown"), title: "Unknown".to_string(), url: hls_url.to_string(), shared: false, diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index 3526a83f7..d8d8545bd 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -126,7 +126,7 @@ async fn m3u_api_stream( let input = try_option_bad_request!( app_state .app_config - .get_input_by_name(pli.input_name.as_str()), + .get_input_by_name(&pli.input_name), true, format!("Cant find input {} for target {target_name}, stream_id {virtual_id}", pli.input_name) ); diff --git a/backend/src/api/endpoints/user_api.rs b/backend/src/api/endpoints/user_api.rs index 03a8646bc..48fab2dc3 100644 --- a/backend/src/api/endpoints/user_api.rs +++ b/backend/src/api/endpoints/user_api.rs @@ -1,20 +1,20 @@ +use crate::api::api_utils::try_unwrap_body; use crate::api::api_utils::{get_user_target_by_username, get_username_from_auth_header}; use crate::api::model::AppState; use crate::auth::validator_user; -use crate::model::{AppConfig, ConfigTarget}; -use shared::model::{PlaylistBouquetDto, TargetType, XtreamCluster}; +use crate::auth::AuthBearer; use crate::model::PlaylistXtreamCategory; +use crate::model::{AppConfig, ConfigTarget}; +use crate::repository::m3u_repository; use crate::repository::user_repository::{load_user_bouquet_as_json, save_user_bouquet}; use crate::repository::xtream_repository::xtream_get_playlist_categories; -use crate::repository::m3u_repository; +use axum::response::IntoResponse; use bytes::Bytes; use futures::{stream, StreamExt}; use log::error; +use shared::model::{PlaylistBouquetDto, TargetType, XtreamCluster}; use std::collections::HashSet; use std::sync::Arc; -use axum::response::IntoResponse; -use crate::auth::AuthBearer; -use crate::api::api_utils::try_unwrap_body; fn get_categories_from_xtream(categories: Option>) -> Vec { let mut groups: Vec = Vec::new(); @@ -27,12 +27,12 @@ fn get_categories_from_xtream(categories: Option>) - } -async fn get_categories_from_m3u_playlist(target: &ConfigTarget, config: &AppConfig) -> Vec { +async fn get_categories_from_m3u_playlist(target: &ConfigTarget, config: &AppConfig) -> Vec> { let mut groups = Vec::new(); if let Some((_guard, iter)) = m3u_repository::iter_raw_m3u_playlist(config, target).await { let mut unique_groups = HashSet::new(); for (item, _has_next) in iter { - if !unique_groups.contains(item.group.as_str()) { + if !unique_groups.contains(&item.group) { unique_groups.insert(item.group.clone()); groups.push(item.group.clone()); } @@ -153,7 +153,7 @@ pub fn user_api_register(app_state: Arc) -> axum::Router .route("/playlist/categories", axum::routing::get(playlist_categories)) .route("/playlist/bouquet", axum::routing::get(playlist_bouquet)) .route("/playlist/bouquet", axum::routing::post(save_playlist_bouquet)) - .route_layer(axum::middleware::from_fn_with_state(app_state, validator_user)) + .route_layer(axum::middleware::from_fn_with_state(app_state, validator_user)), ) diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index 1e010bab7..e06f103aa 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -91,7 +91,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State { let reader = Cursor::new(content); let mut geoip = GeoIp::new(); diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 2c82855ea..9622d0e2c 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -148,7 +148,7 @@ async fn config_batch_content( if let Some(batch_url) = config_input.t_batch_url.as_ref() { let input_source = InputSource::from(&*config_input).with_url(batch_url.to_owned()); let disabled_headers = app_state.get_disabled_headers(); - return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await { + return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None, false).await { Ok((content, _path)) => { // Return CSV with explicit content-type try_unwrap_body!(axum::response::Response::builder() diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index 1d5c9f864..a4be32d06 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1,4 +1,5 @@ // https://github.com/tellytv/go.xtream-codes/blob/master/structs.go +// Xtream api -> https://9tzx6f0ozj.apidog.io/ use crate::api::api_utils; use crate::api::api_utils::{create_session_fingerprint, local_stream_response, try_unwrap_body}; use crate::api::api_utils::{ @@ -32,9 +33,7 @@ use futures::Stream; use log::{debug, error, warn}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use shared::error::create_tuliprox_error_result; -use shared::error::info_err; -use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, info_err, TuliproxError}; use shared::model::{create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType, TargetType, UserConnectionPermission, XtreamCluster}; use shared::utils::{ deserialize_as_string, extract_extension_from_url, generate_playlist_uuid, @@ -92,11 +91,7 @@ impl FromStr for ApiStreamContext { Self::MOVIE => Ok(Self::Movie), Self::SERIES => Ok(Self::Series), Self::TIMESHIFT => Ok(Self::Timeshift), - _ => create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "Unknown ApiStreamContext: {}", - s - ), + _ => info_err_res!("Unknown ApiStreamContext: {}", s), } } } @@ -255,7 +250,7 @@ async fn xtream_player_api_stream( } let input = try_option_bad_request!( - app_state.app_config.get_input_by_name(pli.input_name.as_str()), + app_state.app_config.get_input_by_name(&pli.input_name), true, format!( "Cant find input {} for target {target_name}, context {}, stream_id {virtual_id}", pli.input_name, stream_req.context) ); @@ -447,7 +442,7 @@ async fn xtream_player_api_stream_with_token( let input = try_option_bad_request!( app_state .app_config - .get_input_by_name(pli.input_name.as_str()), + .get_input_by_name(&pli.input_name), true, format!( "Cant find input {} for target {target_name}, context {}, stream_id {}", @@ -829,7 +824,7 @@ async fn xtream_get_stream_info_response( Some(cluster), ).await { if pli.item_type.is_local() { - let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name))) else { + let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected: xtream output required for target {}", target.name)) else { return try_unwrap_body!(empty_json_response()); }; @@ -840,7 +835,7 @@ async fn xtream_get_stream_info_response( if pli.provider_id > 0 { let input_name = &pli.input_name; - if let Some(input) = app_state.app_config.get_input_by_name(input_name.as_str()) { + if let Some(input) = app_state.app_config.get_input_by_name(input_name) { if let Some(info_url) = xtream::get_xtream_player_api_info_url(&input, cluster, pli.provider_id) { // Redirect is only possible for live streams, vod and series info needs to be modified if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live { @@ -925,7 +920,7 @@ async fn xtream_get_short_epg( if pli.provider_id > 0 { let input_name = &pli.input_name; - if let Some(input) = app_state.app_config.get_input_by_name(input_name.as_str()) { + if let Some(input) = app_state.app_config.get_input_by_name(input_name) { if let Some(action_url) = xtream::get_xtream_player_api_action_url( &input, crate::model::XC_ACTION_GET_SHORT_EPG, @@ -952,6 +947,7 @@ async fn xtream_get_short_epg( &input_source, None, None, + false ) .await { @@ -1039,7 +1035,7 @@ async fn xtream_get_catchup_response( ).await); let input = try_option_bad_request!(app_state .app_config - .get_input_by_name(pli.input_name.as_str())); + .get_input_by_name(&pli.input_name)); let info_url = try_option_bad_request!(xtream::get_xtream_player_api_action_url( &input, crate::model::XC_ACTION_GET_CATCHUP_TABLE @@ -1053,7 +1049,8 @@ async fn xtream_get_catchup_response( let content = try_result_bad_request!( xtream::get_xtream_stream_info_content( &app_state.http_client.load(), - &input_source + &input_source, + false ) .await ); @@ -1288,10 +1285,7 @@ async fn xtream_player_api( ) .await ), - _ => Some(Err(info_err!(format!( - "Cant find content: {action} for target: {}", - &target.name - )))), + _ => Some(info_err_res!("Cant find content: {action} for target: {}", &target.name)), }; match result { diff --git a/backend/src/api/model/active_user_manager.rs b/backend/src/api/model/active_user_manager.rs index bcb71966a..3088f1360 100644 --- a/backend/src/api/model/active_user_manager.rs +++ b/backend/src/api/model/active_user_manager.rs @@ -7,7 +7,7 @@ use arc_swap::ArcSwapOption; use jsonwebtoken::get_current_timestamp; use log::{debug, info}; use shared::model::{ActiveUserConnectionChange, StreamChannel, StreamInfo, UserConnectionPermission, VirtualId}; -use shared::utils::{current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, sanitize_sensitive_info, strip_port}; +use shared::utils::{current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, intern, sanitize_sensitive_info, strip_port}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; @@ -251,7 +251,7 @@ impl ActiveUserManager { if &stream.addr == addr { stream.provider = String::from("tuliprox"); stream.channel.title = video_type.to_string(); - stream.channel.group = String::new(); + stream.channel.group = intern(""); return Some(stream.clone()); } } diff --git a/backend/src/api/model/model_utils.rs b/backend/src/api/model/model_utils.rs index 718d4be61..f20c2ea29 100644 --- a/backend/src/api/model/model_utils.rs +++ b/backend/src/api/model/model_utils.rs @@ -5,13 +5,12 @@ use reqwest::header::HeaderMap; use shared::utils::{filter_response_header}; pub fn get_response_headers(headers: &HeaderMap) -> Vec<(String, String)> { - let mut response_headers: Vec<(String, String)> = headers.iter() + let response_headers: Vec<(String, String)> = headers.iter() .filter(|(key, _)| filter_response_header(key.as_str())) .filter_map(|(key, value)| { value.to_str().ok().map(|v| (key.to_string(), v.to_string())) }) .collect(); - response_headers.push((axum::http::header::CONNECTION.as_str().to_string(), "close".to_string())); response_headers } @@ -32,7 +31,6 @@ pub fn get_stream_response_with_headers(custom: Option<(Vec<(String, String)>, S let default_headers = vec![ ("content-type", "application/octet-stream"), - ("connection", "close"), ]; for (key, value) in default_headers { diff --git a/backend/src/api/model/xtream.rs b/backend/src/api/model/xtream.rs index e26835283..fbf940dbb 100644 --- a/backend/src/api/model/xtream.rs +++ b/backend/src/api/model/xtream.rs @@ -7,19 +7,19 @@ use shared::utils::CONSTANTS; #[derive(Serialize, Deserialize, Clone)] pub struct XtreamUserInfoResponse { - pub password: String, pub username: String, - pub active_cons: String, - pub allowed_output_formats: Vec, + pub password: String, + pub message: String, pub auth: u16, // 0 | 1 - #[serde(serialize_with ="serialize_number_as_string")] - pub created_at: i64, //1623429679, + pub status: String, // "Active" #[serde(serialize_with ="serialize_number_as_string")] pub exp_date: i64, //1628755200, pub is_trial: String, // 0 | 1 + pub active_cons: String, + #[serde(serialize_with ="serialize_number_as_string")] + pub created_at: i64, //1623429679, pub max_connections: String, - pub message: String, - pub status: String, // "Active" + pub allowed_output_formats: Vec, } #[derive(Serialize, Deserialize, Clone)] @@ -32,6 +32,7 @@ pub struct XtreamServerInfoResponse { pub timezone: String, pub timestamp_now: i64, pub time_now: String, //"2021-06-28 17:07:37" + pub process: bool, } #[derive(Serialize, Deserialize, Clone)] @@ -72,17 +73,17 @@ impl XtreamAuthorizationResponse { Self { user_info: XtreamUserInfoResponse { - active_cons: format!("{active_connections}"), - allowed_output_formats: CONSTANTS.allowed_output_formats.clone(), + username: user.username.clone(), + password: user.password.clone(), + message: server_info.message.clone(), auth: 1, - created_at, + status: user_status.to_string(), exp_date, is_trial, + active_cons: active_connections.to_string(), + created_at, max_connections, - message: server_info.message.clone(), - password: user.password.clone(), - username: user.username.clone(), - status: user_status.to_string(), + allowed_output_formats: CONSTANTS.allowed_output_formats.clone(), }, server_info: XtreamServerInfoResponse { url: server_info.host.clone(), @@ -93,6 +94,8 @@ impl XtreamAuthorizationResponse { timezone: server_info.timezone.clone(), timestamp_now: now.timestamp(), time_now: now.format("%Y-%m-%d %H:%M:%S").to_string(), + // We don't know what this field is good for, but it is in the response from XtreamCodes, so we will include it. + process: true }, } } diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index f4bb32fe8..1e948ba60 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -5,7 +5,7 @@ use crate::utils::{debug_if_enabled, persist_source_config, read_sources_file_fr use crate::utils::get_csv_file_path; use log::{debug, error, warn}; use serde_json::Value; -use shared::error::{create_tuliprox_error_result, info_err, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, info_err, TuliproxError}; use shared::model::{ConfigInputAliasDto, InputType}; use shared::utils::{get_credentials_from_url, parse_timestamp, sanitize_sensitive_info, trim_last_slash}; use std::path::{Path, PathBuf}; @@ -88,17 +88,17 @@ fn resolve_query_params( if value.eq_ignore_ascii_case("auto") { if key.eq_ignore_ascii_case("api_key") { let Some(k) = api_key.filter(|s| !s.trim().is_empty()) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but panel_api.api_key is missing"); + return info_err_res!("panel_api: query param {key} uses 'auto' but panel_api.api_key is missing"); }; value = k.to_string(); } else if key.eq_ignore_ascii_case("username") { let Some((u, _)) = creds else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but no account username is available"); + return info_err_res!("panel_api: query param {key} uses 'auto' but no account username is available"); }; value = u.to_string(); } else if key.eq_ignore_ascii_case("password") { let Some((_, pw)) = creds else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but no account password is available"); + return info_err_res!("panel_api: query param {key} uses 'auto' but no account password is available"); }; value = pw.to_string(); } @@ -109,7 +109,7 @@ fn resolve_query_params( } fn build_panel_url(base_url: &str, query_params: &[(String, String)]) -> Result { - let mut url = Url::parse(base_url).map_err(|e| info_err!(format!("panel_api: invalid url {base_url}: {e}")))?; + let mut url = Url::parse(base_url).map_err(|e| info_err!("panel_api: invalid url {base_url}: {e}"))?; { let mut pairs = url.query_pairs_mut(); for (k, v) in query_params { @@ -156,14 +156,14 @@ async fn panel_get_json(app_state: &AppState, url: Url) -> Result Result< let url = build_panel_url(cfg.url.as_str(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new response is not a JSON object/array"); + return info_err_res!("panel_api: client_new response is not a JSON object/array"); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new status=false"); + return info_err_res!("panel_api: client_new status=false"); } if let Some((u, p)) = extract_username_password_from_json(obj) { return Ok((u, p, None)); @@ -191,7 +191,7 @@ async fn panel_client_new(app_state: &AppState, cfg: &PanelApiConfig) -> Result< return Ok((u, p, base)); } } - create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new response missing username/password (and no parsable url)") + info_err_res!("panel_api: client_new response missing username/password (and no parsable url)") } async fn panel_client_renew(app_state: &AppState, cfg: &PanelApiConfig, username: &str, password: &str) -> Result<(), TuliproxError> { @@ -203,11 +203,11 @@ async fn panel_client_renew(app_state: &AppState, cfg: &PanelApiConfig, username let url = build_panel_url(cfg.url.as_str(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_renew response is not a JSON object/array"); + return info_err_res!("panel_api: client_renew response is not a JSON object/array"); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_renew status=false"); + return info_err_res!("panel_api: client_renew status=false"); } Ok(()) } @@ -221,11 +221,11 @@ async fn panel_client_info(app_state: &AppState, cfg: &PanelApiConfig, username: let url = build_panel_url(cfg.url.as_str(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_info response is not a JSON object/array"); + return info_err_res!("panel_api: client_info response is not a JSON object/array"); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_info status=false"); + return info_err_res!("panel_api: client_info status=false"); } let expire = obj.get("expire").and_then(|v| v.as_str()).unwrap_or_default().trim(); let parsed = parse_timestamp(expire).ok().flatten(); @@ -302,11 +302,11 @@ async fn patch_source_yml_add_alias( ) -> Result<(), TuliproxError> { let mut sources = match read_sources_file_from_path(source_file_path, false, false, None) { Ok(sources) => sources, - Err(e) => return Err(TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read source file: {e}"))), + Err(e) => return info_err_res!("panel_api: failed to read source file: {e}"), }; let Some(input) = sources.inputs.iter_mut().find(|i| i.name == input_name) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find input '{input_name}' in source.yml"); + return info_err_res!("panel_api: could not find input '{input_name}' in source.yml"); }; let alias = ConfigInputAliasDto { @@ -335,15 +335,15 @@ async fn patch_source_yml_update_exp_date( ) -> Result<(), TuliproxError> { let mut sources = match read_sources_file_from_path(source_file_path, false, false, None) { Ok(sources) => sources, - Err(e) => return Err(TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read source file: {e}"))), + Err(e) => return info_err_res!("panel_api: failed to read source file: {e}"), }; let Some(input) = sources.inputs.iter_mut().find(|i| i.name == input_name) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find input '{input_name}' in source.yml"); + return info_err_res!("panel_api: could not find input '{input_name}' in source.yml"); }; if input.update_account_expiration_date(input_name, account_name, exp_date).is_err() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find account '{account_name}' under input '{input_name}' in source.yml"); + return info_err_res!("panel_api: could not find account '{account_name}' under input '{input_name}' in source.yml"); } persist_source_config(app_state, Some(source_file_path), sources).await?; @@ -399,7 +399,7 @@ async fn patch_batch_csv_append( lines.push(record.join(";")); tokio::fs::write(csv_path, lines.join("\n") + "\n") .await - .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to write csv: {e}")))?; + .map_err(|e| info_err!("panel_api: failed to write csv: {e}"))?; Ok(()) } @@ -412,11 +412,11 @@ async fn patch_batch_csv_update_exp_date( ) -> Result<(), TuliproxError> { let raw = tokio::fs::read_to_string(csv_path) .await - .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read csv: {e}")))?; + .map_err(|e| info_err!("panel_api: failed to read csv: {e}"))?; let mut lines: Vec = raw.lines().map(ToString::to_string).collect(); let header_line_idx = lines.iter().position(|l| l.trim_start().starts_with('#')); let Some(header_idx) = header_line_idx else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: csv missing header line"); + return info_err_res!("panel_api: csv missing header line"); }; let header = lines[header_idx].trim_start_matches('#').trim(); let cols: Vec = header.split(';').map(|s| s.trim().to_lowercase()).collect(); @@ -461,7 +461,7 @@ async fn patch_batch_csv_update_exp_date( lines[i] = fields.join(";"); tokio::fs::write(csv_path, lines.join("\n") + "\n") .await - .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to write csv: {e}")))?; + .map_err(|e| info_err!("panel_api: failed to write csv: {e}"))?; return Ok(()); } } diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index aa3ed211d..e1c4038c4 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -128,6 +128,7 @@ mod tests { use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; use std::path::PathBuf; use std::time::Duration; + use shared::utils::intern; use crate::library::{MediaClassification, MediaClassifier}; fn create_test_config(tmdb_enabled: bool) -> LibraryConfig { @@ -157,8 +158,8 @@ mod tests { formats: vec![], }, playlist: LibraryPlaylistConfig { - movie_category: "Movies".to_string(), - series_category: "Series".to_string(), + movie_category: intern("Movies"), + series_category: intern("Series"), }, } } diff --git a/backend/src/library/scanner.rs b/backend/src/library/scanner.rs index cd4ebb238..5bc88a62a 100644 --- a/backend/src/library/scanner.rs +++ b/backend/src/library/scanner.rs @@ -282,6 +282,7 @@ impl LibraryScanner { #[cfg(test)] mod tests { use super::*; + use shared::utils::intern; use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; fn create_test_config() -> LibraryConfig { @@ -311,8 +312,8 @@ mod tests { formats: vec![], }, playlist: LibraryPlaylistConfig { - movie_category: "Local Movies".to_string(), - series_category: "Local Series".to_string(), + movie_category: intern("Local Movies"), + series_category: intern("Local Series"), }, } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 87182603f..c54f23f56 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -150,13 +150,13 @@ async fn main() { } let app_config = utils::read_initial_app_config(&mut config_paths, true, true, args.server) .await - .unwrap_or_else(|err| exit!("{}", err)); + .unwrap_or_else(|err| exit!("{err}")); print_info(&app_config); let sources = > as Access>::load(&app_config.sources); let targets = sources .validate_targets(args.target.as_ref()) - .unwrap_or_else(|err| exit!("{}", err)); + .unwrap_or_else(|err| exit!("{err}")); if args.server { start_in_server_mode(Arc::new(app_config), Arc::new(targets)).await; diff --git a/backend/src/model/config/app.rs b/backend/src/model/config/app.rs index 207eca44a..0fd4def99 100644 --- a/backend/src/model/config/app.rs +++ b/backend/src/model/config/app.rs @@ -5,7 +5,7 @@ use arc_swap::access::Access; use arc_swap::{ArcSwap, ArcSwapOption}; use log::{error, warn}; use rand::Rng; -use shared::create_tuliprox_error_result; +use shared::info_err_res; use shared::error::{TuliproxError, TuliproxErrorKind}; use shared::model::ConfigPaths; use std::borrow::Cow; @@ -81,10 +81,10 @@ impl AppConfig { if let Some(username) = output_username { if let Some((_, config_target)) = self.get_target_for_username(username) { if config_target.name != target_name { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "User:{username} does not belong to target: {}", target_name); + return info_err_res!("User:{username} does not belong to target: {}", target_name); } } else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "User: {username} does not exist"); + return info_err_res!("User: {username} does not exist"); } Ok(()) } else { @@ -249,20 +249,20 @@ impl AppConfig { for input in &sources.inputs { let input_name = input.name.trim(); if input_name.is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input name required"); + return info_err_res!("input name required"); } if seen_names.contains(input_name) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input names should be unique: {}", input_name); + return info_err_res!("input names should be unique: {}", input_name); } seen_names.insert(input_name.to_string()); if let Some(aliases) = &input.aliases { for alias in aliases { let input_name = alias.name.trim().to_string(); if input_name.is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input name required"); + return info_err_res!("input name required"); } if seen_names.contains(&input_name) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input and alias names should be unique: {}", input_name); + return info_err_res!("input and alias names should be unique: {}", input_name); } seen_names.insert(input_name.clone()); } @@ -280,7 +280,7 @@ impl AppConfig { if let Some(targets) = &schedule.targets { for target_name in targets { if !target_names.contains(target_name.as_str()) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Unknown target name in scheduler: {}", target_name); + return info_err_res!("Unknown target name in scheduler: {}", target_name); } } } diff --git a/backend/src/model/config/base.rs b/backend/src/model/config/base.rs index 91cf2d17d..3712377fd 100644 --- a/backend/src/model/config/base.rs +++ b/backend/src/model/config/base.rs @@ -58,6 +58,7 @@ pub struct Config { pub sleep_timer_mins: Option, pub update_on_boot: bool, pub config_hot_reload: bool, + pub disk_based_processing: bool, pub accept_insecure_ssl_certificates: bool, pub web_ui: Option, pub messaging: Option, @@ -134,6 +135,7 @@ impl From<&ConfigDto> for Config { fn from(dto: &ConfigDto) -> Self { Config { process_parallel: dto.process_parallel, + disk_based_processing: dto.disk_based_processing, api: ConfigApi::from(&dto.api), working_dir: dto.working_dir.clone(), backup_dir: dto.backup_dir.clone(), diff --git a/backend/src/model/config/favourites.rs b/backend/src/model/config/favourites.rs index aa47343d0..09000ab45 100644 --- a/backend/src/model/config/favourites.rs +++ b/backend/src/model/config/favourites.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use crate::model::macros; use regex::Regex; use shared::foundation::filter::{CompiledRegex, Filter}; @@ -5,7 +6,7 @@ use shared::model::{ConfigFavouritesDto, ItemField}; #[derive(Debug, Clone)] pub struct ConfigFavourites { - pub group: String, + pub group: Arc, pub filter: Filter, pub match_as_ascii: bool, } diff --git a/backend/src/model/config/input.rs b/backend/src/model/config/input.rs index 229c9c466..daeee59b0 100644 --- a/backend/src/model/config/input.rs +++ b/backend/src/model/config/input.rs @@ -5,7 +5,7 @@ use log::warn; use shared::error::TuliproxError; use shared::model::{ConfigInputAliasDto, ConfigInputDto, ConfigInputOptionsDto, InputFetchMethod, InputType, StagedInputDto}; use shared::utils::{get_credentials_from_url}; -use shared::{check_input_connections, info_err, write_if_some}; +use shared::{check_input_connections, info_err_res, write_if_some}; use shared::check_input_credentials; use std::collections::HashMap; use std::fmt; @@ -156,7 +156,7 @@ impl ConfigInput { if let Some(staged_input) = &mut self.staged { check_input_credentials!(staged_input, staged_input.input_type, false, true); if !matches!(staged_input.input_type, InputType::M3u | InputType::Xtream) { - return Err(info_err!("Staged input can only be from type m3u or xtream".to_owned())); + return info_err_res!("Staged input can only be from type m3u or xtream"); } } diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index 1ef46c5c0..1c6b5ce6e 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -1,9 +1,9 @@ use crate::model::macros; -use shared::error::TuliproxError; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; use std::path::PathBuf; -use shared::info_err; -use shared::utils::default_metadata_path; +use std::sync::Arc; +use shared::utils::{default_metadata_path, intern}; #[derive(Debug, Clone, Default)] pub struct LibraryScanDirectory { @@ -40,8 +40,8 @@ pub struct LibraryTmdbConfig { #[derive(Debug, Clone)] pub struct LibraryPlaylistConfig { - pub movie_category: String, - pub series_category: String, + pub movie_category: Arc, + pub series_category: Arc, } @@ -67,7 +67,7 @@ impl LibraryConfig { dir.path = path.to_string(); } } - Err(err) => return Err(info_err!(format!("Failed to canonicalize directory path: {err}"))), + Err(err) => return info_err_res!("Failed to canonicalize directory path: {err}"), } } } @@ -121,8 +121,8 @@ impl From<&LibraryConfigDto> for LibraryConfig { formats: dto.metadata.formats.clone(), }, playlist: LibraryPlaylistConfig { - movie_category: dto.playlist.movie_category.clone(), - series_category: dto.playlist.series_category.clone(), + movie_category: intern(&dto.playlist.movie_category), + series_category: intern(&dto.playlist.series_category), }, } } diff --git a/backend/src/model/config/panel_api.rs b/backend/src/model/config/panel_api.rs index feb8ce6b1..9fff392b8 100644 --- a/backend/src/model/config/panel_api.rs +++ b/backend/src/model/config/panel_api.rs @@ -1,6 +1,5 @@ use crate::model::macros; -use shared::create_tuliprox_error_result; -use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{PanelApiConfigDto, PanelApiQueryParamDto, PanelApiQueryParametersDto}; #[derive(Debug, Clone)] @@ -66,24 +65,18 @@ impl PanelApiQueryParameters { .map(|p| p.value.trim().to_string()); match typ { Some(v) if v.eq_ignore_ascii_case("m3u") => Ok(()), - Some(v) => create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: unsupported type={v}, only m3u is supported"), - None => create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: missing required query param 'type=m3u'"), + Some(v) => info_err_res!("panel_api: unsupported type={v}, only m3u is supported"), + None => info_err_res!("panel_api: missing required query param 'type=m3u'"), } } fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); let Some(api_key) = api_key else { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} must contain query param 'api_key' (use value 'auto')" - ); + return info_err_res!("panel_api: {section} must contain query param 'api_key' (use value 'auto')"); }; if api_key.value.trim().is_empty() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" - ); + return info_err_res!("panel_api: {section} query param 'api_key' must not be empty (use value 'auto')"); } Ok(()) } @@ -92,18 +85,12 @@ impl PanelApiQueryParameters { let username = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("username")); let password = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("password")); if username.is_none() || password.is_none() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')" - ); + return info_err_res!("panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')"); } if !username.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) || !password.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)" - ); + return info_err_res!("panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)"); } Ok(()) } @@ -118,7 +105,7 @@ impl PanelApiQueryParameters { Self::require_api_key_param(params, "query_parameter.client_new")?; Self::validate_type_is_m3u(params)?; if params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("user")) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new must not contain query param 'user'"); + return info_err_res!("panel_api: client_new must not contain query param 'user'"); } Ok(()) } @@ -176,19 +163,16 @@ impl PanelApiConfig { return Ok(()); } if self.url.trim().is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: url is missing"); + return info_err_res!("panel_api: url is missing"); } if self.api_key.as_ref().is_none_or(|k| k.trim().is_empty()) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: api_key is missing"); + return info_err_res!("panel_api: api_key is missing"); } if self.query_parameter.client_info.is_empty() || self.query_parameter.client_new.is_empty() || self.query_parameter.client_renew.is_empty() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: query_parameter.client_info/client_new/client_renew must be configured" - ); + return info_err_res!("panel_api: query_parameter.client_info/client_new/client_renew must be configured"); } self.query_parameter.prepare() } diff --git a/backend/src/model/config/source.rs b/backend/src/model/config/source.rs index ca73821e9..5272b3755 100644 --- a/backend/src/model/config/source.rs +++ b/backend/src/model/config/source.rs @@ -1,5 +1,5 @@ use crate::model::{macros, ConfigInput, ConfigTarget, ProcessTargets}; -use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, TuliproxError}; use shared::model::{ConfigSourceDto, PatternTemplate, SourcesConfigDto}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; @@ -67,7 +67,7 @@ impl TryFrom<&SourcesConfigDto> for SourcesConfig { // Validate that all input references exist for input_name in &source_dto.inputs { if !input_names.contains(input_name) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Source references unknown input: {input_name}"); + return info_err_res!("Source references unknown input: {input_name}"); } } sources.push(ConfigSource::from_dto(source_dto)?); @@ -140,7 +140,7 @@ impl SourcesConfig { let missing_targets: Vec = check_targets.iter().filter(|&(_, v)| *v == 0).map(|(k, _)| k.clone()).collect(); if !missing_targets.is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "No target found for {}", missing_targets.join(", ")); + return info_err_res!("No target found for {}", missing_targets.join(", ")); } // let processing_targets: Vec = check_targets.iter().filter(|&(_, v)| *v != 0).map(|(k, _)| k.to_string()).collect(); // info!("Processing targets {}", processing_targets.join(", ")); diff --git a/backend/src/model/config/web_auth.rs b/backend/src/model/config/web_auth.rs index 6fb8083a9..192813a3f 100644 --- a/backend/src/model/config/web_auth.rs +++ b/backend/src/model/config/web_auth.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::BufRead; use std::path::PathBuf; -use shared::error::{TuliproxError, TuliproxErrorKind, create_tuliprox_error_result}; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{UserCredential, WebAuthConfigDto}; use crate::model::macros; use crate::utils; @@ -51,7 +51,7 @@ impl WebAuthConfig { if !utils::path_exists(&userfile_path) { userfile_path = PathBuf::from(config_path).join(&userfile_name); if !utils::path_exists(&userfile_path) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Could not find userfile {}", &userfile_name); + return info_err_res!("Could not find userfile {}", &userfile_name); } } @@ -72,7 +72,7 @@ impl WebAuthConfig { self.t_users = Some(users); } else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Could not read userfile {:?}", &userfile_path); + return info_err_res!("Could not read userfile {:?}", &userfile_path); } Ok(()) } diff --git a/backend/src/model/playlist.rs b/backend/src/model/playlist.rs index f20481bb5..e1a2204b7 100644 --- a/backend/src/model/playlist.rs +++ b/backend/src/model/playlist.rs @@ -1,21 +1,60 @@ -use crate::model::{TVGuide, ConfigInput}; -use shared::model::{PlaylistGroup}; +use std::collections::HashSet; +use crate::model::{ConfigInput, TVGuide}; +use shared::model::{PlaylistGroup, PlaylistItem, UUIDType}; +use crate::repository::PlaylistSource; -#[derive(Debug, Clone)] -pub struct FetchedPlaylist<'a> { // Contains playlist for one input +pub struct FetchedPlaylist<'a> { pub input: &'a ConfigInput, - pub playlist_groups: Vec, + pub source: Box, pub epg: Option, } impl FetchedPlaylist<'_> { - pub fn update_playlist(&mut self, plg: &PlaylistGroup) { - for grp in &mut self.playlist_groups { - if grp.id == plg.id { - plg.channels.iter().for_each(|item| grp.channels.push(item.clone())); - return; - } + pub fn sort_by_provider_ordinal(&mut self) { + self.source.sort_by_provider_ordinal(); + } + + pub async fn update_playlist(&mut self, plg: &PlaylistGroup) { + if self.source.is_memory() { + self.source.update_playlist(plg).await; } } + + pub fn is_memory(&self) -> bool { + self.source.is_memory() + } + + pub fn get_channel_count(&mut self) -> usize { + self.source.get_channel_count() + } + + pub fn get_group_count(&mut self) -> usize { + self.source.get_group_count() + } + + pub fn items_mut(&mut self) -> Box + Send + '_> { + self.source.items_mut() + } + + pub(crate) fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + self.source.items() + } + + pub fn get_missing_vod_info_count(&mut self) -> usize { + self.source.get_missing_vod_info_count() + } + + pub fn get_missing_series_info_count(&mut self) -> usize { + self.source.get_missing_series_info_count() + } + + pub fn deduplicate(&mut self, duplicates: &mut HashSet) { + self.source.deduplicate(duplicates); + } + + pub fn clone_source(&self) -> Box { + self.source.clone_box() + } } + diff --git a/backend/src/processing/mod.rs b/backend/src/processing/mod.rs index f9d8bc5c0..83cbd1f46 100644 --- a/backend/src/processing/mod.rs +++ b/backend/src/processing/mod.rs @@ -1,4 +1,5 @@ -mod playlist_watch; +pub(crate) mod input_cache; +pub(crate) mod playlist_watch; pub(crate) mod parser; pub(crate) mod processor; -pub(crate) mod input_cache; + diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 4dc42eb86..c12bf95c8 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -1,9 +1,10 @@ use crate::model::{Config, ConfigInput}; use crate::utils::request::DynReader; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use shared::utils::{default_supported_video_extensions, extract_id_from_url}; +use shared::utils::{default_supported_video_extensions, extract_id_from_url, StringInterner}; use std::borrow::BorrowMut; use tokio::io::AsyncBufReadExt; +use indexmap::IndexMap; // other implementations like calculating text_distance on all titles took too much time // we keep it now as simple as possible and less memory intensive. @@ -88,11 +89,11 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { } } -fn create_empty_playlistitem_header(input_name: &str, url: String) -> PlaylistItemHeader { +fn create_empty_playlistitem_header(interner: &mut StringInterner, input_name: &str, url: String) -> PlaylistItemHeader { PlaylistItemHeader { url, category_id: 0, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), ..Default::default() } } @@ -108,7 +109,7 @@ macro_rules! process_header_fields { }; } -fn process_header(input_name: &str, video_suffixes: &[String], content: &str, url: String) -> PlaylistItemHeader { +fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixes: &[String], content: &str, url: String) -> PlaylistItemHeader { let url_id = extract_id_from_url(&url); let url_types = if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { // TODO find Series based on group or configured names @@ -117,7 +118,7 @@ fn process_header(input_name: &str, video_suffixes: &[String], content: &str, ur None }; - let mut plih = create_empty_playlistitem_header(input_name, url); + let mut plih = create_empty_playlistitem_header(interner, input_name, url); let mut it = content.chars(); let mut stack = String::with_capacity(64); let line_token = token_till(&mut stack, &mut it, ':', false); @@ -147,14 +148,16 @@ fn process_header(input_name: &str, video_suffixes: &[String], content: &str, ur } else { process_header_fields!(plih, token.as_str(), (id, "tvg-id"), - (group, "group-title"), (name, "tvg-name"), (parent_code, "parent-code"), (audio_track, "audio-track"), (logo, "tvg-logo"), (logo_small, "tvg-logo-small"), (time_shift, "timeshift"), - (rec, "tvg-rec"); value); + (rec, "tvg-rec"); value.clone()); + if token == "group-title" { + plih.group = interner.intern(&value); + } } } } @@ -200,6 +203,7 @@ fn process_header(input_name: &str, video_suffixes: &[String], content: &str, ur pub async fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader, mut visit: F) { let mut header: Option = None; let mut group: Option = None; + let mut interner = StringInterner::new(); let input_name = input.name.as_str(); let video_suffixes = match cfg.video.as_ref() { @@ -209,6 +213,7 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp None => default_supported_video_extensions() }; let mut lines = tokio::io::BufReader::new(lines).lines(); + let mut ord_counter: u32 = 1; while let Ok(Some(line)) = lines.next_line().await { if line.starts_with("#EXTINF") { header = Some(line); @@ -222,14 +227,16 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp continue; } if let Some(header_value) = header { - let mut item = PlaylistItem { header: process_header(input_name, &video_suffixes, &header_value, line) }; + let mut item = PlaylistItem { header: process_header(&mut interner, input_name, &video_suffixes, &header_value, line) }; let header = &mut item.header; + header.source_ordinal = ord_counter; + ord_counter += 1; if header.group.is_empty() { if let Some(group_value) = group { - header.group = group_value; + header.group = interner.intern(&group_value); } else { let current_title = header.title.clone(); - header.group = get_title_group(current_title.as_str()); + header.group = interner.intern(&get_title_group(current_title.as_str())); } } visit(item); @@ -241,30 +248,17 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> Vec { - let mut sort_order: Vec> = vec![]; - let mut sort_order_idx: usize = 0; - let mut group_map: std::collections::HashMap = std::collections::HashMap::new(); + let mut group_map: IndexMap> = IndexMap::new(); consume_m3u(cfg, input, lines, |item| { - // keep the original sort order for groups and group the playlist items let key = { let header = &item.header; format!("{}{}", &header.xtream_cluster, &header.group) }; - match group_map.entry(key) { - std::collections::hash_map::Entry::Vacant(v) => { - v.insert(sort_order_idx); - sort_order.push(vec![item]); - sort_order_idx += 1; - } - std::collections::hash_map::Entry::Occupied(o) => { - if let Some(order) = sort_order.get_mut(*o.get()) { - order.push(item); - } - } - } + group_map.entry(key).or_default().push(item); }).await; + let mut grp_id = 0; - let result: Vec = sort_order.into_iter().filter_map(|channels| { + group_map.into_values().filter_map(|channels| { // create a group based on the first playlist item let channel = channels.first(); if let Some((cluster, group_title)) = channel.map(|pli| @@ -274,55 +268,58 @@ pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> V } else { None } - }).collect(); - result + }).collect() } #[cfg(test)] mod test { use crate::processing::parser::m3u::process_header; + use shared::utils::StringInterner; #[test] fn test_process_header_1() { + let mut interner = StringInterner::new(); let input: &str = "hello"; let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney" , Seven"#; - let pli = process_header(input, &video_suffixes, line, url.to_string()); + let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "abc-seven"); assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); assert_eq!(pli.chno, 7); - assert_eq!(pli.group, "Sydney"); + assert_eq!(&*pli.group, "Sydney"); } #[test] fn test_process_header_2() { + let mut interner = StringInterner::new(); let input: &str = "hello"; let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney", Seven"#; - let pli = process_header(input, &video_suffixes, line, url.to_string()); + let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "abc-seven"); assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); assert_eq!(pli.chno, 7); - assert_eq!(pli.group, "Sydney"); + assert_eq!(&*pli.group, "Sydney"); } #[test] fn test_process_header_xui_id() { + let mut interner = StringInterner::new(); let input: &str = "hello"; let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 tvg-id="abc-seven" xui-id="provider-123" group-title="Sydney", Seven"#; - let pli = process_header(input, &video_suffixes, line, url.to_string()); + let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "provider-123"); // Should use xui-id assert_eq!(pli.epg_channel_id, Some("abc-seven".to_string())); // Should preserve original tvg-id - assert_eq!(pli.group, "Sydney"); + assert_eq!(&*pli.group, "Sydney"); } } \ No newline at end of file diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 149265676..9a6650ec0 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -3,11 +3,17 @@ use crate::model::ConfigInput; use crate::model::{XtreamCategory}; use crate::utils::request::DynReader; use crate::utils::xtream::get_xtream_stream_url_base; -use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind}; -use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, XtreamCluster}; -use shared::utils::{generate_playlist_uuid, trim_last_slash}; +use shared::error::{notify_err_res, notify_err, TuliproxError}; +use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, + PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, + SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, + XtreamCluster, XtreamPlaylistItem}; +use shared::utils::{generate_playlist_uuid, trim_last_slash, StringInterner}; use indexmap::IndexMap; use tokio::task::spawn_blocking; +use serde::Deserializer; + + async fn map_to_xtream_category(categories: DynReader) -> Result, TuliproxError> { spawn_blocking(move || { @@ -15,10 +21,10 @@ async fn map_to_xtream_category(categories: DynReader) -> Result>(reader) { Ok(xtream_categories) => Ok(xtream_categories), Err(err) => { - create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process categories {}", &err) + notify_err_res!("Failed to process categories {}", &err) } } - }).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream categories failed: {e}")))? + }).await.map_err(|e| notify_err!("Mapping xtream categories failed: {e}"))? } async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader) -> Result, TuliproxError> { @@ -39,10 +45,10 @@ async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader Ok(stream_list) } Err(err) => { - create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {xtream_cluster}: {err}", ) + notify_err_res!("Failed to map to xtream streams {xtream_cluster}: {err}") } } - }).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream streams failed: {e}")))? + }).await.map_err(|e| notify_err!("Mapping xtream streams failed: {e}"))? } fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, password: &'a str, episode: &'a SeriesStreamDetailEpisodeProperties) -> Cow<'a, str> { @@ -55,7 +61,9 @@ fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, passwor } } -pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, group_title: &str, series_name: &str, input: &ConfigInput) -> Option> { + pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, + group_title: &str, series_name: &str, input: &ConfigInput, + interner: &mut StringInterner) -> Option> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); @@ -72,14 +80,14 @@ pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStre parent_code: parent_uuid.to_string(), name: series_name.to_string(), logo: episode.movie_image.clone(), - group: group_title.to_string(), + group: interner.intern(group_title), title: episode.title.clone(), url: episode_url.to_string(), item_type: PlaylistItemType::Series, xtream_cluster: XtreamCluster::Series, additional_properties: Some(StreamProperties::Episode(episode_info)), category_id: 0, - input_name: input.name.clone(), + input_name: interner.intern(&input.name), ..Default::default() } } @@ -129,6 +137,7 @@ pub async fn parse_xtream(input: &ConfigInput, streams: DynReader) -> Result>, TuliproxError> { match map_to_xtream_category(categories).await { Ok(xtream_categories) => { + let mut interner = StringInterner::new(); let input_name = input.name.clone(); let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); @@ -147,18 +156,20 @@ pub async fn parse_xtream(input: &ConfigInput, let (live_stream_use_prefix, live_stream_without_extension) = input.options.as_ref() .map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension)); + // Re-implement the loop to add ordinal + let mut ord_counter: u32 = 1; for stream in xtream_streams { let group = group_map.get_mut(&stream.get_category_id()).unwrap_or(&mut unknown_grp); let category_name = &group.category_name; let stream_url = create_xtream_url(xtream_cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); let item_type = PlaylistItemType::from(xtream_cluster); - let item = PlaylistItem { + let mut item = PlaylistItem { header: PlaylistItemHeader { id: stream.get_stream_id().to_string(), uuid: generate_playlist_uuid(&input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), name: stream.get_name().to_string(), logo: stream.get_stream_icon().to_string(), - group: category_name.clone(), + group: interner.intern(category_name), title: stream.get_name().to_string(), url: stream_url.to_string(), epg_channel_id: stream.get_epg_channel_id(), @@ -166,10 +177,12 @@ pub async fn parse_xtream(input: &ConfigInput, xtream_cluster, additional_properties: Some(stream), category_id: 0, - input_name: input_name.clone(), + input_name: interner.intern(&input_name), ..Default::default() }, }; + item.header.source_ordinal = ord_counter; + ord_counter += 1; group.add(item); } @@ -184,7 +197,7 @@ pub async fn parse_xtream(input: &ConfigInput, PlaylistGroup { id: category.category_id, xtream_cluster, - title: category.category_name.clone(), + title: interner.intern(&category.category_name), channels: category.channels.clone(), } }).collect())) @@ -196,6 +209,167 @@ pub async fn parse_xtream(input: &ConfigInput, } } +pub async fn parse_xtream_streaming( + input: &ConfigInput, + xtream_cluster: XtreamCluster, + categories: DynReader, + streams: DynReader, + mut on_item: F, +) -> Result, TuliproxError> +where + F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> + Send + 'static +{ + // 1. Parse Categories + let xtream_categories = map_to_xtream_category(categories).await?; + + // 2. Prepare for Stream Parsing + let mut interner = StringInterner::new(); + let input_name = input.name.clone(); + let url = input.url.as_str().to_string(); + let username = input.username.as_ref().map_or("", |v| v).to_string(); + let password = input.password.as_ref().map_or("", |v| v).to_string(); + let options = input.options.clone(); + + // Map categories for lookup + let group_map: IndexMap = xtream_categories.iter().map(|c| (c.category_id, c.category_name.clone())).collect(); + let unknown_group_name = "Unknown".to_string(); + + spawn_blocking(move || { + let reader = tokio_util::io::SyncIoBridge::new(streams); + let mut deserializer = serde_json::Deserializer::from_reader(reader); + + let (live_stream_use_prefix, live_stream_without_extension) = options.as_ref() + .map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension)); + + let mut source_ordinal = 0u32; + + match xtream_cluster { + XtreamCluster::Live => { + let mut on_stream = |stream: LiveStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Live(stream); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &mut interner, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + XtreamCluster::Video => { + let mut on_stream = |stream: VideoStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Video(Box::new(stream)); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &mut interner, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + XtreamCluster::Series => { + let mut on_stream = |stream: SeriesStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Series(Box::new(stream)); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &mut interner, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + } + Ok(()) + }).await.map_err(|e| notify_err!("Streaming parse failed: {e}"))??; + + Ok(xtream_categories) +} + +#[allow(clippy::too_many_arguments)] +fn process_stream_item( + input_name: &str, url: &str, username: &str, password: &str, + cluster: XtreamCluster, interner: &mut StringInterner, + group_map: &IndexMap, unknown_group_name: &str, + mut stream: StreamProperties, + callback: &mut F, + live_stream_use_prefix: bool, live_stream_without_extension: bool, + source_ordinal: u32, +) -> Result<(), TuliproxError> +where F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> +{ + stream.prepare(); + let category_id = stream.get_category_id(); + let category_name = group_map.get(&category_id).map_or(unknown_group_name, String::as_str); + let stream_url = create_xtream_url(cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); + + let item_type = PlaylistItemType::from(cluster); + let item = PlaylistItem { + header: PlaylistItemHeader { + id: stream.get_stream_id().to_string(), + uuid: generate_playlist_uuid(input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), + name: stream.get_name().to_string(), + logo: stream.get_stream_icon().to_string(), + group: interner.intern(category_name), + title: stream.get_name().to_string(), + url: stream_url.to_string(), + epg_channel_id: stream.get_epg_channel_id(), + item_type, + xtream_cluster: cluster, + additional_properties: Some(stream), + category_id, + source_ordinal, + input_name: interner.intern(input_name), + ..Default::default() + }, + }; + + // if let Some(StreamProperties::Series(props)) = item.header.additional_properties.as_mut() { + // // We need to set category_id for Series properties just like parse_xtream might expect or use? + // // Actually parse_xtream doesn't modify internal category_ids, but mapping to XtreamCategory struct relies on it. + // // Here we are creating PlaylistItem. + // let _ = props; + // } + + callback(XtreamPlaylistItem::from(&item)) +} + +struct XtreamItemVisitor<'a, T, F> { + on_item: &'a mut F, + _marker: std::marker::PhantomData, +} + +impl<'de, T, F> serde::de::Visitor<'de> for XtreamItemVisitor<'_, T, F> +where + T: serde::Deserialize<'de>, + F: FnMut(T) -> Result<(), TuliproxError>, +{ + type Value = (); + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a JSON array or an error object") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + while let Some(item) = seq.next_element::()? { + (self.on_item)(item).map_err(serde::de::Error::custom)?; + } + Ok(()) + } + + fn visit_map(self, map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let val: serde_json::Value = serde::de::Deserialize::deserialize(serde::de::value::MapAccessDeserializer::new(map))?; + if let Some(msg) = val.get("message").and_then(|m| m.as_str()) { + return Err(serde::de::Error::custom(format!("Xtream API error: {msg}"))); + } + Err(serde::de::Error::custom(format!("Expected array, got object: {val}"))) + } +} + #[cfg(test)] mod tests { use crate::processing::parser::xtream::map_to_xtream_streams; @@ -237,4 +411,20 @@ mod tests { } Ok(()) } -} \ No newline at end of file + + #[test] + fn test_xtream_item_visitor_array() { + use serde_json::Deserializer; + use shared::model::LiveStreamProperties; + let data = r#"[{"name":"stream1", "stream_id": 1, "category_id": 1, "added": "0"}]"#; + let mut deserializer = Deserializer::from_str(data); + let mut count = 0; + let mut on_item = |_: LiveStreamProperties| { + count += 1; + Ok(()) + }; + let visitor = super::XtreamItemVisitor { on_item: &mut on_item, _marker: std::marker::PhantomData }; + serde::Deserializer::deserialize_any(&mut deserializer, visitor).unwrap(); + assert_eq!(count, 1); + } +} diff --git a/backend/src/processing/processor/epg.rs b/backend/src/processing/processor/epg.rs index 400be91ed..98895cf80 100644 --- a/backend/src/processing/processor/epg.rs +++ b/backend/src/processing/processor/epg.rs @@ -1,8 +1,8 @@ use crate::model::{Epg, TVGuide, XmlTag, XmlTagIcon, EPG_ATTRIB_ID}; use crate::model::{EpgConfig, EpgSmartMatchConfig}; -use crate::model::{FetchedPlaylist}; +use crate::model::FetchedPlaylist; use crate::processing::parser::xmltv::normalize_channel_name; -use log::{debug, trace}; +use log::{debug, trace, warn}; use rphonetic::{DoubleMetaphone, Encoder}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; @@ -109,10 +109,12 @@ impl EpgIdCache<'_> { let smart_match_enabled = self.smart_match_enabled; let fuzzy_matching = self.fuzzy_match_enabled; - for channel in fp.playlist_groups.iter().flat_map(|g| &g.channels) { + // Helper closure to process a single item + // We use a closure here to capture `self` and avoid code duplication + let mut process_item = |name: &str, epg_channel_id: Option<&str>| { let mut missing_epg_id = true; // insert epg_id to known channel epg_ids - if let Some(id) = channel.header.epg_channel_id.as_deref() { + if let Some(id) = epg_channel_id { if !id.is_empty() { missing_epg_id = false; self.channel_epg_id.insert(Cow::Owned(id.to_string())); @@ -124,8 +126,13 @@ impl EpgIdCache<'_> { let needs_normalization = smart_match_enabled && (fuzzy_matching || missing_epg_id); if needs_normalization { - let name = &channel.header.name; - self.normalize_and_store(name, channel.header.epg_channel_id.as_deref()); + self.normalize_and_store(name, epg_channel_id); + } + }; + + for channel in fp.items() { + if channel.header.xtream_cluster == XtreamCluster::Live && channel.header.item_type.is_live() { + process_item(&channel.header.name, channel.header.epg_channel_id.as_deref()); } } } @@ -211,11 +218,13 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> } }; - let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live; - fp.playlist_groups.iter_mut() - .flat_map(|g| &mut g.channels) - .filter(filter_live) - .for_each(assign_values); + let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live && c.header.item_type.is_live(); + + if fp.is_memory() { + fp.items_mut().filter(filter_live).for_each(assign_values); + } else { + warn!("Disk based playlist modification is not supported!"); + } processed_epgs.push(epg_source); } } @@ -238,6 +247,9 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> /// process_playlist_epg(&mut playlist, &mut epg_data); /// ``` pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec) { + if fp.input.epg.is_none() { + return; + } // collect all epg_channel ids let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); id_cache.collect_epg_id(fp); diff --git a/backend/src/processing/processor/library.rs b/backend/src/processing/processor/library.rs index 5461062d4..734bc1c79 100644 --- a/backend/src/processing/processor/library.rs +++ b/backend/src/processing/processor/library.rs @@ -2,7 +2,7 @@ use crate::library::{MediaMetadata, MetadataAsyncIter, MetadataCacheEntry}; use crate::model::{AppConfig, ConfigInput}; use shared::error::TuliproxError; use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster}; -use shared::utils::{generate_playlist_uuid}; +use shared::utils::{generate_playlist_uuid, StringInterner}; use std::path::Path; use std::sync::Arc; @@ -25,15 +25,14 @@ pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &A channels: vec![], xtream_cluster: XtreamCluster::Series, }; + let mut interner = StringInterner::new(); while let Some(entry) = metadata_iter.next().await { match entry.metadata { MediaMetadata::Movie(_) => { - let pli = to_playlist_item(&entry, &input.name, &library_config.playlist.movie_category); - group_movies.channels.extend(pli); + to_playlist_item(&mut interner, &entry, &input.name, &library_config.playlist.movie_category, &mut group_movies.channels); } MediaMetadata::Series(_) => { - let pli = to_playlist_item(&entry, &input.name, &library_config.playlist.series_category); - group_series.channels.extend(pli); + to_playlist_item(&mut interner, &entry, &input.name, &library_config.playlist.series_category, &mut group_series.channels); } } } @@ -49,30 +48,30 @@ pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &A (groups, vec![]) } -fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &str) -> Vec { + +fn to_playlist_item(interner: &mut StringInterner, entry: &MetadataCacheEntry, input_name: &str, group_name: &str, channels: &mut Vec) { let metadata = &entry.metadata; match metadata { MediaMetadata::Movie(_) => { let additional_properties = metadata_cache_entry_to_xtream_movie_info(entry); - vec![PlaylistItem { + channels.push(PlaylistItem { header: PlaylistItemHeader { uuid: UUIDType::from_valid_uuid(&entry.uuid), name: metadata.title().to_string(), - group: group_name.to_string(), + group: interner.intern(group_name), title: metadata.title().to_string(), logo: metadata.poster().map_or_else(String::new, ToString::to_string), url: format!("file://{}", entry.file_path), xtream_cluster: XtreamCluster::Video, additional_properties, item_type: PlaylistItemType::LocalVideo, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), ..PlaylistItemHeader::default() } - }] + }); } MediaMetadata::Series(_series) => { - let mut items = vec![]; if let Some(additional_properties) = metadata_cache_entry_to_xtream_series_info(entry) { let mut episodes = vec![]; if let StreamProperties::Series(series_properties) = &additional_properties { @@ -92,13 +91,13 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s uuid: generate_playlist_uuid(input_name, &episode.id.to_string(), PlaylistItemType::LocalSeries, &episode.direct_source), logo: logo.clone(), name: episode.title.clone(), - group: group_name.to_string(), + group: interner.intern(group_name), title: episode.title.clone(), url: episode.direct_source.clone(), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeries, category_id: 0, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), additional_properties: Some(StreamProperties::Episode(EpisodeStreamProperties { episode_id: episode.id, episode: episode.episode_num, @@ -124,23 +123,20 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s uuid: UUIDType::from_valid_uuid(&entry.uuid), id: entry.uuid.clone(), name: metadata.title().to_string(), - group: group_name.to_string(), + group: interner.intern(group_name), title: metadata.title().to_string(), logo: metadata.poster().map_or_else(String::new, ToString::to_string), url: format!("file://{}", entry.file_path), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeriesInfo, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), additional_properties: Some(additional_properties), ..PlaylistItemHeader::default() } }; - - items.push(series_info); - items.extend(episodes); + channels.push(series_info); + channels.extend(episodes); } - - items } } } diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 4fe2768c3..0d53e8203 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -1,16 +1,18 @@ -use crate::model::{AppConfig, Config, ConfigFavourites, ConfigInput, ConfigRename}; +use crate::model::{AppConfig, Config, ConfigFavourites, ConfigInput, ConfigRename, TVGuide}; use crate::utils::m3u; use crate::utils::xtream; use crate::utils::{epg, StepMeasureCallback}; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::Mutex; +use std::sync::{Arc, Weak}; +use tokio::sync::{Mutex, OwnedRwLockWriteGuard, RwLock}; use tokio::task::JoinSet; use crate::api::model::{EventManager, EventMessage, PlaylistStorageState, UpdateGuard}; use crate::messaging::send_message_json; use crate::model::Epg; + + use crate::model::FetchedPlaylist; use crate::model::Mapping; use crate::model::{ConfigTarget, ProcessTargets}; @@ -26,16 +28,20 @@ use crate::processing::processor::trakt::process_trakt_categories_for_target; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::{load_input_playlist, persist_input_playlist, persist_playlist}; +use crate::repository::xtream_repository::CategoryKey; +use crate::repository::{MemoryPlaylistSource, PlaylistSource}; use crate::utils::StepMeasure; use crate::utils::{debug_if_enabled, trace_if_enabled}; use futures::StreamExt; use indexmap::IndexMap; -use log::{debug, error, info, log_enabled, trace, warn, Level}; +use log::{debug, error, info, log_enabled, warn, Level}; use shared::error::{get_errors_notify_message, notify_err, TuliproxError}; use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider}; use shared::model::xtream_const::XTREAM_CLUSTER; -use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster}; -use shared::utils::{create_alias_uuid, default_as_default}; +use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, + PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState, + ProcessingOrder, UUIDType}; +use shared::utils::{create_alias_uuid, default_as_default, StringInterner}; use std::time::Instant; fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { @@ -43,14 +49,36 @@ fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { filter.filter(&provider) } -#[allow(clippy::unnecessary_wraps)] +pub fn apply_filter_to_source(source: &mut dyn PlaylistSource, filter: &Filter) -> Option> { + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); + for pli in source.into_items() { + if is_valid(&pli, filter, false) { + let group_title = pli.header.group.clone(); + let cluster = pli.header.xtream_cluster; + let cat_id = pli.header.category_id; + groups.entry(group_title.clone()) + .or_insert_with(|| PlaylistGroup { + id: cat_id, + title: group_title, + channels: vec![], + xtream_cluster: cluster, + }) + .channels.push(pli); + } + } + + if groups.is_empty() { None } else { Some(groups.into_values().collect()) } +} + +fn filter_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, _interner: &mut StringInterner) -> Option> { + apply_filter_to_source(source, &target.filter) +} + pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) -> Option> { - debug!("Filtering {} groups", playlist.len()); let mut new_playlist = Vec::with_capacity(128); for pg in playlist.iter_mut() { let channels = pg.channels.iter() .filter(|&pli| is_valid(pli, filter, false)).cloned().collect::>(); - trace!("Filtered group {} has now {}/{} items", pg.title, channels.len(), pg.channels.len()); if !channels.is_empty() { new_playlist.push(PlaylistGroup { id: pg.id, @@ -60,11 +88,7 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) }); } } - Some(new_playlist) -} - -fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { - apply_filter_to_playlist(playlist, &target.filter) + if new_playlist.is_empty() { None } else { Some(new_playlist) } } fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { @@ -86,7 +110,7 @@ fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { } } -fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>) { +fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>, interner: &mut StringInterner) { if let Some(renames) = rename { if !renames.is_empty() { let result = pli; @@ -97,33 +121,41 @@ fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>) { trace_if_enabled!("Renamed {}={value} to {cap}", &r.field); } let value = cap.into_owned(); - set_field_value(result, r.field, value); + set_field_value(result, r.field, value, interner); } } } } -fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { +fn rename_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, interner: &mut StringInterner) -> Option> { match &target.rename { - Some(renames) => { - if !renames.is_empty() { - let mut new_playlist: Vec = Vec::with_capacity(playlist.len()); - for g in playlist { - let mut grp = g.clone(); - for r in renames { - if matches!(r.field, ItemField::Group) { - let cap = r.pattern.replace_all(&grp.title, &r.new_name); - trace_if_enabled!("Renamed group {} to {cap} for {}", &grp.title, target.name); - grp.title = cap.into_owned(); + Some(renames) if !renames.is_empty() => { + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); + for mut pli in source.into_items() { + // Handle group rename first if it's in the renames + for r in renames { + if matches!(r.field, ItemField::Group) { + let value = &*pli.header.group; + let cap = r.pattern.replace_all(value, &r.new_name); + if *value != cap { + pli.header.group = interner.intern(&cap); } } - - grp.channels.iter_mut().for_each(|pli| exec_rename(pli, target.rename.as_ref())); - new_playlist.push(grp); } - return Some(new_playlist); + exec_rename(&mut pli, Some(renames), interner); + let group_title = pli.header.group.clone(); + let cluster = pli.header.xtream_cluster; + let cat_id = pli.header.category_id; + groups.entry(group_title.clone()) + .or_insert_with(|| PlaylistGroup { + id: cat_id, + title: group_title, + channels: vec![], + xtream_cluster: cluster, + }) + .channels.push(pli); } - None + Some(groups.into_values().collect()) } _ => None } @@ -164,41 +196,34 @@ fn map_channel_and_flatten(channel: PlaylistItem, mapping: &Mapping) -> Vec Option> { - if let Some(mappings) = target.mapping.load().as_ref() { - let new_playlist: Vec = playlist.iter().map(|playlist_group| { - let mut grp = playlist_group.clone(); - mappings.iter().filter(|&mapping| mapping.mapper.as_ref().is_some_and(|v| !v.is_empty())) - .for_each(|mapping| - grp.channels = grp.channels.drain(..).flat_map(|chan| map_channel_and_flatten(chan, mapping)).collect()); - grp - }).collect(); - - // if the group names are changed, restructure channels to the right groups - // we use - let mut new_groups: Vec = Vec::with_capacity(128); - let mut grp_id: u32 = 0; - for playlist_group in new_playlist { - for channel in &playlist_group.channels { - let cluster = &channel.header.xtream_cluster; - let title = &channel.header.group; - if let Some(grp) = new_groups.iter_mut().find(|x| *x.title == **title) { - grp.channels.push(channel.clone()); - } else { - grp_id += 1; - new_groups.push(PlaylistGroup { - id: grp_id, - title: title.clone(), - channels: vec![channel.clone()], - xtream_cluster: *cluster, - }); +fn map_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, _interner: &mut StringInterner) -> Option> { + let mapping_binding = target.mapping.load(); + let mappings = mapping_binding.as_ref()?; + let valid_mappings = mappings.iter().filter(|m| m.mapper.as_ref().is_some_and(|v| !v.is_empty())); + let iter: Box> = Box::new(source.into_items()); + let mapped_iter = valid_mappings.fold(iter, |iter, mapping| { + Box::new(iter.flat_map(move |chan| map_channel_and_flatten(chan, mapping))) + as Box> + }); + let mut next_groups: IndexMap, PlaylistGroup> = IndexMap::new(); + let mut grp_id: u32 = 0; + for channel in mapped_iter { + let group_title = channel.header.group.clone(); + let cluster = channel.header.xtream_cluster; + next_groups.entry(group_title.clone()) + .or_insert_with(|| { + grp_id += 1; + PlaylistGroup { + id: grp_id, + title: group_title, + channels: Vec::new(), + xtream_cluster: cluster, } - } - } - Some(new_groups) - } else { - None + }) + .channels.push(channel); } + + Some(next_groups.into_values().collect()) } fn map_playlist_counter(target: &ConfigTarget, playlist: &mut [PlaylistGroup]) { @@ -247,7 +272,7 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo (!user_targets.enabled && target.enabled) || (user_targets.enabled && user_targets.has_target(target.id)) } -async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc, input: &ConfigInput) -> (Vec, Vec, bool) { +async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc, input: &ConfigInput) -> (Vec, Vec, bool, bool) { let config = &*app_config.config.load(); let working_dir = &config.working_dir; @@ -286,14 +311,20 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc }; if fully_cached { - return (vec![], vec![], true); + return (vec![], vec![], true, false); } - let (playlist, errors) = match input.input_type { - InputType::M3u => m3u::download_m3u_playlist(client, config, input).await, + let (playlist, errors, persisted) = match input.input_type { + InputType::M3u => { + let (p, e) = m3u::download_m3u_playlist(client, config, input).await; + (p, e, false) + } InputType::Xtream => xtream::download_xtream_playlist(config, client, input, clusters_to_download.as_deref()).await, - InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![]), - InputType::Library => library::download_library_playlist(client, app_config, input).await, + InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![], false), + InputType::Library => { + let (p, e) = library::download_library_playlist(client, app_config, input).await; + (p, e, false) + } }; // Update Status @@ -330,20 +361,18 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc } } - (playlist, errors, false) + (playlist, errors, false, persisted) } -async fn process_source(client: &reqwest::Client, app_config: Arc, source_idx: usize, - user_targets: Arc, event_manager: Option>, - playlist_state: Option<&Arc>, -) -> (Vec, Vec, Vec) { - let sources = app_config.sources.load(); +async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (Vec, Vec, Vec) { + let sources = ctx.config.sources.load(); let mut errors = vec![]; let mut input_stats = HashMap::::new(); let mut target_stats = Vec::::new(); if let Some(source) = sources.get_source_at(source_idx) { + let mut interner = StringInterner::default(); let mut source_playlists = Vec::with_capacity(128); - let broadcast_step = create_broadcast_callback(event_manager.as_ref()); + let broadcast_step = create_broadcast_callback(ctx.event_manager.as_ref()); // Download the sources let mut source_downloaded = false; for input_name in &source.inputs { @@ -351,25 +380,15 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so error!("Input {input_name} referenced by source {source_idx} does not exist"); continue; }; - if is_input_enabled(input, &user_targets) { + if is_input_enabled(input, &ctx.user_targets) { source_downloaded = true; let start_time = Instant::now(); // Download the playlist for input - let (playlist_groups, mut error_list) = { + let (mut playlist_groups, mut error_list) = { broadcast_step("Playlist download", &format!("Downloading input '{}'", input.name)); - // Caching Logic - let (downloaded_playlist, mut download_err, was_cached) = playlist_download_from_input(client, &app_config, input).await; - let (playlist, error) = if was_cached { - match load_input_playlist(&app_config, input, None).await { - Ok(pl) => (pl, None), - Err(e) => (vec![], Some(e)), - } - } else { - broadcast_step("Playlist download", &format!("Persisting input '{}' playlist", input.name)); - persist_input_playlist(&app_config, input, downloaded_playlist).await - }; + let (mut download_err, playlist, error) = download_input(ctx, input).await; if let Some(err) = error { broadcast_step("Playlist download", &format!("Failed to persist/load input '{}' playlist", input.name)); @@ -379,60 +398,46 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so (playlist, download_err) }; - // Download epg for input - let (tvguide, mut tvguide_errors) = if error_list.is_empty() { - broadcast_step("Playlist download", &format!("Downloading epg for input '{}'", input.name)); - let working_dir = &app_config.config.load().working_dir; - epg::get_xmltv(client, input, working_dir).await + let (tvguide, mut tvguide_errors) = if input.input_type == InputType::Library { + (None, vec!()) } else { - (None, vec![]) + download_input_epg(ctx,input, &mut errors).await }; errors.append(&mut error_list); errors.append(&mut tvguide_errors); - let group_count = playlist_groups.len(); - let channel_count = playlist_groups.iter().map(|group| group.channels.len()).sum(); + let group_count = playlist_groups.get_group_count(); + let channel_count = playlist_groups.get_channel_count(); let input_name = &input.name; if playlist_groups.is_empty() { broadcast_step("Playlist download", &format!("Input '{}' playlist is empty", input.name)); info!("Source is empty {input_name}"); - errors.push(notify_err!(format!("Source is empty {input_name}"))); + errors.push(notify_err!("Source is empty {input_name}")); } else { source_playlists.push( FetchedPlaylist { input, - // If I create Arc here, it drops at end of loop iteration! - // SAFETY ISSUE. - // `source_playlists` stores `FetchedPlaylist`. - // `FetchedPlaylist` struct definition: - // pub struct FetchedPlaylist<'a> { pub input: &'a ConfigInput, ... } - // It holds a REFERENCE. - // If `input` comes from `sources` (guard), it lives as long as `sources` guard lives. - // `sources` guard is alive in this function scope. - // So `&input` is valid. - // `playlist_download_from_input` takes `&Arc`. I pass formatted Arc. - // `FetchedPlaylist` needs reference. - playlist_groups, + source: playlist_groups, epg: tvguide, } ); } let elapsed = start_time.elapsed().as_secs(); - input_stats.insert(input_name.clone(), create_input_stat(group_count, channel_count, error_list.len(), + input_stats.insert(input_name.clone(), create_input_stat(group_count, channel_count, errors.len(), input.input_type, input_name, elapsed)); } } if source_downloaded { if source_playlists.is_empty() { debug!("Source at index {source_idx} is empty"); - errors.push(notify_err!(format!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(std::string::String::as_str).collect::>().join(", ")))); + errors.push(notify_err!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(std::string::String::as_str).collect::>().join(", "))); } else { - debug_if_enabled!("Source has {} groups", source_playlists.iter().map(|fpl| fpl.playlist_groups.len()).sum::()); - let event_manager_clone = event_manager.clone(); + debug_if_enabled!("Source has {} groups", source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::()); for target in &source.targets { - let event_manager_clone = event_manager_clone.clone(); - if is_target_enabled(target, &user_targets) { - match process_playlist_for_target(&app_config, client, &mut source_playlists, target, &mut input_stats, &mut errors, event_manager_clone, playlist_state).await { + if is_target_enabled(target, &ctx.user_targets) { + match process_playlist_for_target(ctx, &mut source_playlists, target, + &mut input_stats, &mut errors, + &mut interner).await { Ok(()) => { target_stats.push(TargetStats::success(&target.name)); } @@ -449,6 +454,58 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so (input_stats.into_values().collect(), target_stats, errors) } +async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc, + error_list: &mut [TuliproxError]) -> (Option, Vec) { + // Download epg for input + let (tvguide, tvguide_errors) = if error_list.is_empty() { + debug!("Downloading epg for input '{}'", input.name); + let working_dir = &ctx.config.config.load().working_dir; + epg::get_xmltv(ctx, input, working_dir).await + } else { + (None, vec![]) + }; + (tvguide, tvguide_errors) +} + +async fn download_input(ctx: &PlaylistProcessingContext, input: &Arc) + -> (Vec, Box, Option) { + // Coordination Logic + let need_download = !ctx.is_input_downloaded(&input.name).await; + + let (downloaded_playlist, download_err, was_cached, persisted) = if need_download { + // Acquire named lock to prevent thundering herd on same input + let _input_lock = ctx.get_input_lock(&input.name).await; + // Check again after lock + let already_processed = ctx.is_input_downloaded(&input.name).await; + + if already_processed { + // Use empty results, will load from disk below + (vec![], vec![], true, false) + } else { + let res = playlist_download_from_input(&ctx.client, &ctx.config, input).await; + // Mark as processed if NO critical errors? + // playlist_download_from_input returns errors but also potentially a partial playlist. + // If it attempted download, we consider it processed for this session. + ctx.mark_input_downloaded(input.name.clone()).await; + res + } + } else { + (vec![], vec![], true, false) + }; + + let (playlist, error) = if was_cached || persisted { + match load_input_playlist(ctx, input, None).await { + Ok(pl_source) => (pl_source, None), + Err(e) => (MemoryPlaylistSource::default().boxed(), Some(e)), + } + } else { + debug!("Persisting input '{}' playlist", input.name); + let (pl, err) = persist_input_playlist(&ctx.config, input, downloaded_playlist).await; + (MemoryPlaylistSource::new(pl).boxed(), err) + }; + (download_err, playlist, error) +} + fn create_broadcast_callback(event_manager: Option<&Arc>) -> StepMeasureCallback { if let Some(event_mgr) = event_manager { let events = event_mgr.clone(); @@ -477,42 +534,80 @@ fn create_input_stat(group_count: usize, channel_count: usize, error_count: usiz } } -async fn process_sources(client: &reqwest::Client, config: &Arc, user_targets: Arc, - event_manager: Option>, playlist_state: Option<&Arc>, -) -> (Vec, Vec) { + +#[derive(Clone)] +pub struct PlaylistProcessingContext { + pub client: reqwest::Client, + pub config: Arc, + pub user_targets: Arc, + pub event_manager: Option>, + pub playlist_state: Option>, + + // Coordination + processed_inputs: Arc>>, + input_locks: Arc>>>>, +} + +impl PlaylistProcessingContext { + pub async fn is_input_downloaded(&self, input_name: &str) -> bool { + let processed = self.processed_inputs.lock().await; + processed.contains(input_name) + } + pub async fn mark_input_downloaded(&self, input_name: String) -> bool { + let mut processed = self.processed_inputs.lock().await; + processed.insert(input_name) + } + + pub async fn get_input_lock(&self, input_name: &str) -> OwnedRwLockWriteGuard<()> { + let mut locks = self.input_locks.lock().await; + // Clean up stale weak references + locks.retain(|_, weak| weak.strong_count() > 0); + + if let Some(weak) = locks.get(input_name) { + if let Some(strong) = weak.upgrade() { + return strong.write_owned().await; + } + } + + let lock = Arc::new(RwLock::new(())); + locks.insert(input_name.to_string(), Arc::downgrade(&lock)); + lock.write_owned().await + } +} + +async fn process_sources(processing_ctx: &PlaylistProcessingContext) -> (Vec, Vec) { let mut async_tasks = JoinSet::new(); - let sources = config.sources.load(); - let process_parallel = config.config.load().process_parallel && sources.sources.len() > 1; + let sources = processing_ctx.config.sources.load(); + let process_parallel = processing_ctx.config.config.load().process_parallel && sources.sources.len() > 1; if process_parallel && log_enabled!(Level::Debug) { debug!("Parallel processing enabled"); } + let errors = Arc::new(Mutex::>::new(vec![])); let stats = Arc::new(Mutex::>::new(vec![])); + for (index, source) in sources.sources.iter().enumerate() { - if !source.should_process_for_user_targets(&user_targets) { + if !source.should_process_for_user_targets(&processing_ctx.user_targets) { continue; } // We're using the file lock this way on purpose let source_lock_path = PathBuf::from(format!("source_{index}")); - let Ok(update_lock) = config.file_locks.try_write_lock(&source_lock_path).await else { + let Ok(update_lock) = processing_ctx.config.file_locks.try_write_lock(&source_lock_path).await else { warn!("The update operation for the source at index {index} was skipped because an update is already in progress."); continue; }; let shared_errors = errors.clone(); let shared_stats = stats.clone(); - let cfg = config.clone(); - let usr_trgts = user_targets.clone(); - let event_manager = event_manager.clone(); + let ctx = processing_ctx.clone(); + if process_parallel { - let http_client = client.clone(); - let playlist_state = playlist_state.cloned(); async_tasks.spawn(async move { // Hold the per-source lock for the full duration of this update. let current_update_lock = update_lock; let (input_stats, target_stats, mut res_errors) = - process_source(&http_client, cfg, index, usr_trgts, event_manager, playlist_state.as_ref()).await; + process_source(index, &ctx).await; shared_errors.lock().await.append(&mut res_errors); if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) { shared_stats.lock().await.push(process_stats); @@ -521,7 +616,7 @@ async fn process_sources(client: &reqwest::Client, config: &Arc, user }); } else { let (input_stats, target_stats, mut res_errors) = - process_source(client, cfg, index, usr_trgts, event_manager, playlist_state).await; + process_source(index, &ctx).await; shared_errors.lock().await.append(&mut res_errors); if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) { shared_stats.lock().await.push(process_stats); @@ -541,7 +636,7 @@ async fn process_sources(client: &reqwest::Client, config: &Arc, user } } -pub type ProcessingPipe = Vec Option>>; +pub type ProcessingPipe = Vec Option>>; fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { match &target.processing_order { @@ -554,29 +649,27 @@ fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { } } -fn duplicate_hash(item: &PlaylistItem) -> UUIDType { - item.get_uuid() -} - fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>, - duplicates: &mut HashSet) -> FetchedPlaylist<'a> { + duplicates: &mut HashSet, + interner: &mut StringInterner) -> FetchedPlaylist<'a> { let mut new_fpl = FetchedPlaylist { input: fpl.input, - playlist_groups: fpl.playlist_groups.clone(), // we need to clone, because of multiple target definitions, we cant change the initial playlist. + source: fpl.clone_source(), epg: fpl.epg.clone(), }; if target.options.as_ref().is_some_and(|opt| opt.remove_duplicates) { - for group in &mut new_fpl.playlist_groups { - // `HashSet::insert` returns true for first insert, otherwise false - group.channels.retain(|item| duplicates.insert(duplicate_hash(item))); - } + new_fpl.deduplicate(duplicates); } for f in pipe { - if let Some(groups) = f(&mut new_fpl.playlist_groups, target) { - new_fpl.playlist_groups = groups; + if let Some(groups) = f(new_fpl.source.as_mut(), target, interner) { + new_fpl.source = MemoryPlaylistSource::new(groups).boxed(); } } + // Ensure source is memory-based for downstream mutable processing (VOD/series resolution) + if !new_fpl.is_memory() { + new_fpl.source = MemoryPlaylistSource::new(new_fpl.source.take_groups()).boxed(); + } new_fpl } @@ -585,9 +678,9 @@ fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedP fn flatten_groups(playlistgroups: Vec) -> Vec { let mut sort_order: Vec = vec![]; let mut idx: usize = 0; - let mut group_map: HashMap<(String, XtreamCluster), usize> = HashMap::new(); + let mut group_map: HashMap = HashMap::new(); for group in playlistgroups { - let key = (group.title.clone(), group.xtream_cluster); + let key = (group.xtream_cluster, group.title.clone()); match group_map.entry(key) { std::collections::hash_map::Entry::Vacant(v) => { v.insert(idx); @@ -605,14 +698,12 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { } #[allow(clippy::too_many_arguments)] -async fn process_playlist_for_target(app_config: &Arc, - client: &reqwest::Client, +async fn process_playlist_for_target(ctx: &PlaylistProcessingContext, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, stats: &mut HashMap, errors: &mut Vec, - event_manager: Option>, - playlist_state: Option<&Arc>, + interner: &mut StringInterner, ) -> Result<(), Vec> { debug_if_enabled!("Processing order is {}", &target.processing_order); @@ -620,22 +711,23 @@ async fn process_playlist_for_target(app_config: &Arc, let mut processed_fetched_playlists: Vec = vec![]; debug!("Executing processing pipes"); - let broadcast_step = create_broadcast_callback(event_manager.as_ref()); + let broadcast_step = create_broadcast_callback(ctx.event_manager.as_ref()); let pipe = get_processing_pipe(target); let mut step = StepMeasure::new(&target.name, broadcast_step); for provider_fpl in playlists.iter_mut() { step.broadcast("Executing transformations on '{}' playlist", &target.name); - let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); - playlist_resolve_series(app_config, client, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; - playlist_resolve_vod(app_config, client, target, errors, &mut processed_fpl).await; + let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates, interner); + processed_fpl.sort_by_provider_ordinal(); + playlist_resolve_series(&ctx.config, &ctx.client, target, errors, &pipe, provider_fpl, &mut processed_fpl, interner).await; + playlist_resolve_vod(&ctx.config, &ctx.client, target, errors, provider_fpl, &mut processed_fpl).await; // stats - let input_stats = stats.get_mut(&processed_fpl.input.name); - if let Some(stat) = input_stats { - stat.processed_stats.group_count = processed_fpl.playlist_groups.len(); - stat.processed_stats.channel_count = processed_fpl.playlist_groups.iter() - .map(|group| group.channels.len()) - .sum(); + let input_entry_name = processed_fpl.input.name.clone(); + let group_count = processed_fpl.get_group_count(); + let channel_count = processed_fpl.get_channel_count(); + if let Some(stat) = stats.get_mut(&input_entry_name) { + stat.processed_stats.group_count = group_count; + stat.processed_stats.channel_count = channel_count; } processed_fetched_playlists.push(processed_fpl); } @@ -654,7 +746,7 @@ async fn process_playlist_for_target(app_config: &Arc, Ok(()) } else { // Process Trakt categories - if trakt_playlist(client, target, errors, &mut new_playlist).await { + if trakt_playlist(&ctx.client, target, errors, &mut new_playlist).await { step.tick("trakt categories"); } @@ -669,11 +761,11 @@ async fn process_playlist_for_target(app_config: &Arc, map_playlist_counter(target, &mut flat_new_playlist); step.tick("assigning channel counter"); - let config = app_config.config.load(); - if process_watch(&config, client, target, &flat_new_playlist).await { + let config = ctx.config.config.load(); + if process_watch(&config, &ctx.client, target, &flat_new_playlist).await { step.tick("group watches"); } - let result = persist_playlist(app_config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, playlist_state).await; + let result = persist_playlist(&ctx.config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, ctx.playlist_state.as_ref(), interner).await; step.stop("Persisting playlists"); result } @@ -681,7 +773,7 @@ async fn process_playlist_for_target(app_config: &Arc, pub fn process_favourites(playlist: &mut Vec, favourites_cfg: Option<&[ConfigFavourites]>) { if let Some(favourites) = favourites_cfg { - let mut fav_groups: IndexMap> = IndexMap::new(); + let mut fav_groups: IndexMap, Vec> = IndexMap::new(); for pg in playlist.iter() { for pli in &pg.channels { // series episodes cant be included in favourites @@ -737,14 +829,14 @@ async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors: } async fn process_epg(processed_fetched_playlists: &mut Vec>) -> (Vec, Vec) { - let mut new_playlist = vec![]; + let mut new_playlist: Vec = vec![]; let mut new_epg = vec![]; // each fetched playlist can have its own epgl url. // we need to process each input epg. for fp in processed_fetched_playlists { process_playlist_epg(fp, &mut new_epg).await; - new_playlist.append(&mut fp.playlist_groups); + new_playlist.extend(fp.source.take_groups()); } (new_epg, new_playlist) } @@ -786,9 +878,19 @@ pub async fn exec_processing(client: &reqwest::Client, app_config: Arc Ordering { - let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.clone() }; - let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.clone() }; + let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.to_string() }; + let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.to_string() }; playlist_comparator(group_sort.sequence.as_ref(), group_sort.order, &value_a, &value_b) } @@ -144,7 +144,7 @@ pub(in crate::processing::processor) fn sort_playlist(target: &ConfigTarget, new } let regexp = &channel_sort.group_pattern; for group in new_playlist.iter_mut() { - let group_title = if match_as_ascii { deunicode(&group.title) } else { group.title.clone() }; + let group_title = if match_as_ascii { deunicode(&group.title) } else { group.title.to_string() }; if regexp.is_match(group_title.as_str()) { group.channels.sort_by(|chan1, chan2| playlistitem_comparator(chan1, chan2, channel_sort, match_as_ascii)); } diff --git a/backend/src/processing/processor/trakt.rs b/backend/src/processing/processor/trakt.rs index b68873355..f4353338f 100644 --- a/backend/src/processing/processor/trakt.rs +++ b/backend/src/processing/processor/trakt.rs @@ -4,6 +4,7 @@ use crate::utils::{extract_year_from_title, normalize_title_for_matching, TraktC use crate::utils::{trace_if_enabled, with}; use log::{debug, info, trace, warn}; use shared::error::TuliproxError; +use shared::utils::StringInterner; use shared::model::{FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, TraktContentType, XtreamCluster}; use shared::utils::CONSTANTS; use std::borrow::Cow; @@ -128,6 +129,7 @@ fn find_best_match_for_item<'a>( fn create_category_from_matches<'a>( matches: Vec>, list_config: &'a TraktListConfig, + interner: &mut StringInterner, ) -> Option { if matches.is_empty() { return None; } @@ -164,7 +166,7 @@ fn create_category_from_matches<'a>( header.set_field("caption", &caption); } } - header.group = String::from(group_title); + header.group = interner.intern(group_title); header.gen_uuid(); }); matched_items.push(modified_item); @@ -184,7 +186,7 @@ fn create_category_from_matches<'a>( Some(PlaylistGroup { id: 0, - title: String::from(group_title), + title: interner.intern(group_title), channels: matched_items, xtream_cluster: cluster, }) @@ -194,6 +196,7 @@ fn match_trakt_items_with_playlist<'a>( trakt_items: &'a [TraktListItem], playlist: &'a [PlaylistGroup], list_config: &'a TraktListConfig, + interner: &mut StringInterner, ) -> Option { let trakt_match_items: Vec> = trakt_items .iter() @@ -217,7 +220,7 @@ fn match_trakt_items_with_playlist<'a>( } } - create_category_from_matches(matches, list_config) + create_category_from_matches(matches, list_config, interner) } pub struct TraktCategoriesProcessor { @@ -244,6 +247,8 @@ impl TraktCategoriesProcessor { info!("Processing {} Trakt lists for target {}", trakt_config.lists.len(), target.name); let mut new_categories = Vec::new(); let mut total_matches = 0; + let mut interner = StringInterner::new(); + for list_config in &trakt_config.lists { let cache_key = format!("{}:{}", list_config.user, list_config.list_slug); @@ -251,7 +256,7 @@ impl TraktCategoriesProcessor { Ok(trakt_items) => { debug!("Processing Trakt list {cache_key} with {} items", trakt_items.len()); - if let Some(category) = match_trakt_items_with_playlist(&trakt_items, playlist, list_config) { + if let Some(category) = match_trakt_items_with_playlist(&trakt_items, playlist, list_config, &mut interner) { if !category.channels.is_empty() { total_matches += category.channels.len(); let category_len = category.channels.len(); diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index 331a93e4a..911bfc172 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -9,10 +9,10 @@ pub(in crate::processing) async fn playlist_resolve_download_playlist_item(clien let provider_id = pli.get_provider_id()?; if let Some(info_url) = xtream::get_xtream_player_api_info_url(input, cluster, provider_id) { let input_source = InputSource::from(input).with_url(info_url); - result = match xtream::get_xtream_stream_info_content(client, &input_source).await { + result = match xtream::get_xtream_stream_info_content(client, &input_source, true).await { Ok(content) => Some(content), Err(err) => { - errors.push(info_err!(format!("{err}"))); + errors.push(info_err!("{err}")); None } }; diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index b7f1fa887..3a10d34f1 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -5,21 +5,28 @@ use crate::processing::processor::create_resolve_options_function_for_xtream_tar use crate::processing::processor::playlist::ProcessingPipe; use crate::processing::processor::xtream::playlist_resolve_download_playlist_item; use crate::repository::storage::get_input_storage_path; -use crate::repository::xtream_repository::persists_input_series_info; +use crate::repository::xtream_repository::persist_input_series_info_batch; +use crate::repository::{MemoryPlaylistSource, PlaylistSource}; use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; use shared::model::{InputType, PlaylistEntry, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo}; use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster}; +use shared::utils::StringInterner; +use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; create_resolve_options_function_for_xtream_target!(series); +const BATCH_SIZE: usize = 100; + +#[allow(clippy::too_many_lines)] async fn playlist_resolve_series_info(app_config: &Arc, client: &reqwest::Client, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_series: bool, - resolve_delay: u16) -> Vec { + resolve_delay: u16, + interner: &mut StringInterner) -> Vec { let input = fpl.input; let working_dir = &app_config.config.load().working_dir; let storage_path = match get_input_storage_path(&input.name, working_dir) { @@ -30,88 +37,96 @@ async fn playlist_resolve_series_info(app_config: &Arc, client: &reqw } }; - let series_info_count = if resolve_series { - let count = fpl.playlist_groups.iter() - .flat_map(|plg| &plg.channels) - .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Series - && pli.header.item_type == PlaylistItemType::SeriesInfo - && !pli.has_details()).count(); - info!("Found {count} series info to resolve"); - count - } else { - 0 - }; + let series_info_count = fpl.get_missing_series_info_count(); + + if series_info_count > 0 { + info!("Found {series_info_count} series info to resolve"); + } let mut last_log_time = Instant::now(); let mut processed_series_info_count = 0; - let mut result: Vec = vec![]; + let mut group_series: HashMap = HashMap::new(); + let mut batch = Vec::with_capacity(BATCH_SIZE); - for plg in &mut fpl.playlist_groups { - let mut group_series = vec![]; - for pli in &mut plg.channels { - if pli.header.xtream_cluster != XtreamCluster::Series - || pli.header.item_type != PlaylistItemType::SeriesInfo { - continue; - } + let input = fpl.input; + for pli in fpl.items_mut() { + if pli.header.xtream_cluster != XtreamCluster::Series + || pli.header.item_type != PlaylistItemType::SeriesInfo { + continue; + } - let Some(provider_id) = pli.get_provider_id() else { continue; }; - if provider_id == 0 { - continue; - } + let Some(provider_id) = pli.get_provider_id() else { continue; }; + if provider_id == 0 { + continue; + } - let should_download = resolve_series && !pli.has_details(); - if should_download { - processed_series_info_count += 1; - if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await { - if !content.is_empty() { - match serde_json::from_str::(&content) { - Ok(info) => { - let series_stream_props = SeriesStreamProperties::from_info(&info, pli); - // the input db needs to be updated - let _ = persists_input_series_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &series_stream_props).await; - // Update in-memory playlist items with the newly fetched vod info. - // This makes the data available for later processing steps like STRM export. - pli.header.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props))); - } - Err(err) => { - error!("Failed to parse series info for provider_id {provider_id}: {err}"); + let should_download = resolve_series && !pli.has_details(); + if should_download { + processed_series_info_count += 1; + if let Some(content) = playlist_resolve_download_playlist_item(client, pli, input, errors, resolve_delay, XtreamCluster::Series).await { + if !content.is_empty() { + match serde_json::from_str::(&content) { + Ok(info) => { + let series_stream_props = SeriesStreamProperties::from_info(&info, pli); + + batch.push((provider_id, series_stream_props.clone())); + if batch.len() >= BATCH_SIZE { + if let Err(err) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, &input.name, std::mem::take(&mut batch)).await { + error!("Failed to persist batch series info: {err}"); + } } + + // Update in-memory playlist items with the newly fetched vod info. + // This makes the data available for later processing steps like STRM export. + pli.header.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props))); + } + Err(err) => { + error!("Failed to parse series info for provider_id {provider_id}: {err}"); } } } } + } - // extract episodes from info - if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() { - let (group, series_name) = { - let header = &pli.header; - (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) - }; - if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input) { - group_series.extend(episodes.into_iter()); - } - } - - if resolve_series && log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { - info!("resolved {processed_series_info_count}/{series_info_count} series info"); - last_log_time = Instant::now(); + // extract episodes from info + if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() { + let (group, series_name) = { + let header = &pli.header; + (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) + }; + if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input, interner) { + let group = group_series.entry(pli.header.category_id) + .or_insert_with(|| { + PlaylistGroup { + id: pli.header.category_id, + title: pli.header.group.clone(), + channels: Vec::new(), + xtream_cluster: XtreamCluster::Series, + } + }); + group.channels.extend(episodes.into_iter()); } } - if !group_series.is_empty() { - result.push(PlaylistGroup { - id: plg.id, - title: plg.title.clone(), - channels: group_series, - xtream_cluster: XtreamCluster::Series, - }); + + if resolve_series && log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_series_info_count}/{series_info_count} series info"); + last_log_time = Instant::now(); } } + + if !batch.is_empty() { + if let Err(err) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, &input.name, batch).await { + error!("Failed to persist final batch series info: {err}"); + } + } + if resolve_series { info!("resolved {processed_series_info_count}/{series_info_count} series info"); } - result + group_series.into_values().collect() } +#[allow(clippy::too_many_arguments)] pub async fn playlist_resolve_series(cfg: &Arc, client: &reqwest::Client, target: &ConfigTarget, @@ -119,25 +134,34 @@ pub async fn playlist_resolve_series(cfg: &Arc, pipe: &ProcessingPipe, provider_fpl: &mut FetchedPlaylist<'_>, processed_fpl: &mut FetchedPlaylist<'_>, + interner: &mut StringInterner, ) { let (resolve_series, resolve_delay) = get_resolve_series_options(target, processed_fpl); - let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay).await; + provider_fpl.source.release_resources(XtreamCluster::Series); + let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay, interner).await; + provider_fpl.source.obtain_resources().await; if series_playlist.is_empty() { return; } - // original content saved into original list - for plg in &series_playlist { - provider_fpl.update_playlist(plg); - } - // run processing pipe over new items - let mut new_playlist = series_playlist; - for f in pipe { - if let Some(v) = f(&mut new_playlist, target) { - new_playlist = v; + if provider_fpl.is_memory() { + // original content saved into original list + for plg in &series_playlist { + provider_fpl.update_playlist(plg).await; } } + // run the processing pipe over new items + let mut new_playlist = series_playlist; + for f in pipe { + let mut source = MemoryPlaylistSource::new(new_playlist); + if let Some(v) = f(&mut source, target, interner) { + new_playlist = v; + } else { + new_playlist = source.take_groups(); + } + } + // assign new items to the new playlist for plg in &new_playlist { - processed_fpl.update_playlist(plg); + processed_fpl.update_playlist(plg).await; } } diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 76a7e7fe8..2b6b792ee 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -3,7 +3,7 @@ use crate::model::{AppConfig, ConfigTarget}; use crate::processing::processor::create_resolve_options_function_for_xtream_target; use crate::processing::processor::xtream::playlist_resolve_download_playlist_item; use crate::repository::storage::get_input_storage_path; -use crate::repository::xtream_repository::persist_input_vod_info; +use crate::repository::xtream_repository::persist_input_vod_info_batch; use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; use shared::model::{InputType, PlaylistEntry, StreamProperties, VideoStreamProperties, XtreamVideoInfo}; @@ -11,10 +11,16 @@ use shared::model::{PlaylistItemType, XtreamCluster}; use std::sync::Arc; use std::time::Instant; + create_resolve_options_function_for_xtream_target!(vod); -pub async fn playlist_resolve_vod(app_config: &Arc, client: &reqwest::Client, - target: &ConfigTarget, errors: &mut Vec, +const BATCH_SIZE: usize = 100; + +pub async fn playlist_resolve_vod(app_config: &Arc, + client: &reqwest::Client, + target: &ConfigTarget, + errors: &mut Vec, + provider_fpl: &mut FetchedPlaylist<'_>, fpl: &mut FetchedPlaylist<'_>) { let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl); if !resolve_movies { return; } @@ -29,50 +35,59 @@ pub async fn playlist_resolve_vod(app_config: &Arc, client: &reqwest: } }; - // LocalVideo entries are not resolved! - let vod_info_count = fpl.playlist_groups.iter() - .flat_map(|plg| &plg.channels) - .filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video - && pli.header.item_type == PlaylistItemType::Video - && !pli.has_details()).count(); + let vod_info_count = fpl.get_missing_vod_info_count(); + info!("Found missing {vod_info_count} vod info to resolve"); - - info!("Found {vod_info_count} vod info to resolve"); let mut last_log_time = Instant::now(); let mut processed_vod_info_count = 0; + let mut batch = Vec::with_capacity(BATCH_SIZE); - for plg in &mut fpl.playlist_groups { - for pli in &mut plg.channels { - if pli.header.xtream_cluster != XtreamCluster::Video - || pli.header.item_type != PlaylistItemType::Video - || pli.has_details() { - continue; - } - let Some(provider_id) = pli.get_provider_id() else { continue; }; - processed_vod_info_count += 1; - if provider_id != 0 { - if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await { - if content.is_empty() { continue; } - match serde_json::from_str::(&content) { - Ok(info) => { - let video_stream_props = VideoStreamProperties::from_info(&info, pli); - if let Err(err) = persist_input_vod_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &video_stream_props).await { - error!("Failed to persist VOD info for provider_id {provider_id}: {err}"); + provider_fpl.source.release_resources(XtreamCluster::Video); + + let input = fpl.input; + for pli in fpl.items_mut() { + if pli.header.xtream_cluster != XtreamCluster::Video + || pli.header.item_type != PlaylistItemType::Video + || pli.has_details() { + continue; + } + let Some(provider_id) = pli.get_provider_id() else { continue; }; + processed_vod_info_count += 1; + if provider_id != 0 { + if let Some(content) = playlist_resolve_download_playlist_item(client, pli, input, errors, resolve_delay, XtreamCluster::Video).await { + if content.is_empty() { continue; } + match serde_json::from_str::(&content) { + Ok(info) => { + let video_stream_props = VideoStreamProperties::from_info(&info, pli); + + batch.push((provider_id, video_stream_props.clone())); + if batch.len() >= BATCH_SIZE { + if let Err(err) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, &input.name, std::mem::take(&mut batch)).await { + error!("Failed to persist batch VOD info: {err}"); } - // This makes the data available for subsequent processing steps like STRM export. - pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props))); - } - Err(err) => { - error!("Failed to parse video info for provider_id {provider_id}: {err}"); } + + // This makes the data available for subsequent processing steps like STRM export. + pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props))); + } + Err(err) => { + error!("Failed to parse video info for provider {} stream_id {provider_id}: {err} {content}", input.name); } } } - if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { - info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); - last_log_time = Instant::now(); - } + } + if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); + last_log_time = Instant::now(); } } + + if !batch.is_empty() { + if let Err(err) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, &input.name, batch).await { + error!("Failed to persist final batch VOD info: {err}"); + } + } + + provider_fpl.source.obtain_resources().await; info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); } diff --git a/backend/src/repository/bplustree.rs b/backend/src/repository/bplustree.rs index 7dbbc4312..d3c4a9852 100644 --- a/backend/src/repository/bplustree.rs +++ b/backend/src/repository/bplustree.rs @@ -1,17 +1,17 @@ -use std::ffi::OsString; use crate::utils; use crate::utils::{binary_deserialize, binary_serialize}; +use fs2::FileExt; use indexmap::IndexMap; use log::error; use serde::{Deserialize, Serialize}; use shared::error::{string_to_io_error, to_io_error}; +use std::ffi::OsString; use std::fs::{File, OpenOptions}; use std::io::{self, BufReader, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; use std::mem::size_of; use std::path::{Path, PathBuf}; use tempfile::NamedTempFile; -use fs2::FileExt; // Constants (Restored) const PAGE_SIZE: u16 = 4096; @@ -231,7 +231,6 @@ impl From for BPlusTreeError { } - impl BPlusTreeError { pub fn to_io(self) -> io::Error { match self { @@ -255,7 +254,6 @@ impl From for BPlusTreeError { impl<'a> SlottedPage<'a> { - pub fn new(data: &'a mut [u8], page_type: PageType) -> Result { if data.len() < PAGE_HEADER_SIZE_USIZE { return Err(PageError::NoSpace); @@ -366,7 +364,7 @@ impl<'a> SlottedPage<'a> { let slot_pos = PAGE_HEADER_SIZE_USIZE + (index * SLOT_SIZE); // Safe slice access if slot_pos + 2 > self.data.len() { return None; } - let offset = u16::from_le_bytes(self.data[slot_pos..slot_pos+2].try_into().ok()?); + let offset = u16::from_le_bytes(self.data[slot_pos..slot_pos + 2].try_into().ok()?); // Bounds check for length header if (offset as usize) + 4 > self.data.len() { return None; } @@ -379,7 +377,7 @@ impl<'a> SlottedPage<'a> { pub fn get_cell_offset(&self, index: usize) -> Option { let slot_pos = PAGE_HEADER_SIZE_USIZE + (index * SLOT_SIZE); if slot_pos + 2 > self.data.len() { return None; } - Some(u16::from_le_bytes(self.data[slot_pos..slot_pos+2].try_into().ok()?)) + Some(u16::from_le_bytes(self.data[slot_pos..slot_pos + 2].try_into().ok()?)) } pub fn compact(&mut self) -> Result<(), PageError> { @@ -433,7 +431,7 @@ impl<'a> SlottedPage<'a> { return Err(PageError::InvalidIndex); // Cannot split empty page } if count == 1 { - // Cannot split single item fundamentally. + // Cannot split single item fundamentally. // Return Ok(None) explicitly to indicate no-op. return Ok(None); } @@ -524,7 +522,7 @@ struct ValueInfo { mode: ValueStorageMode, length: u32, #[serde(skip)] - compressed_cache: Option<(u8, Vec)> // (flag, payload) + compressed_cache: Option<(u8, Vec)>, // (flag, payload) } @@ -664,6 +662,14 @@ where } } + pub fn len(&self) -> usize { + if self.is_leaf { + self.keys.len() + } else { + self.children.iter().map(BPlusTreeNode::len).sum() + } + } + pub fn traverse(&self, visit: &mut F) where F: FnMut(&Vec, &Vec), @@ -1388,6 +1394,14 @@ where } } + pub fn is_empty(&self) -> bool { + self.root.keys.is_empty() + } + + pub fn len(&self) -> usize { + self.root.len() + } + pub fn insert(&mut self, key: K, value: V) { self.dirty = true; if self.root.keys.is_empty() { @@ -1459,6 +1473,97 @@ where } } + /// Bulk build a tree from pre-calculated `ValueInfos` (streaming compact helper). + /// Writes nodes to `file` starting at `start_offset`. + /// Returns the offset of the root node used to update the file header. + fn bulk_build_from_infos( + &mut self, + file: &mut W, + items: &[(K, ValueInfo)], + start_offset: u64, + ) -> io::Result { + if items.is_empty() { + // Write empty root + let root = BPlusTreeNode::::new(true); + let mut buffer = Vec::with_capacity(PAGE_SIZE_USIZE); + let _offset = root.serialize_to_block(file, &mut buffer, start_offset)?; + // For a single empty root, the root offset is the start offset. + // serialize_to_block returns the *end* offset (start of next block). + // But header needs the *start* of the root node. + // Wait - serialize_to_block logic: + // "Returns the offset of the NEXT block" + // So the root IS at start_offset. + return Ok(start_offset); + } + + // 1. Sort items (Query iterator is usually sorted, but let's be safe or rely on caller) + // Since BPlusTreeQuery works on offsets, order might be physical not logical if we iterate blocks? + // BPlusTreeQuery::iter() uses range_scan which traverses logical order. So it IS sorted! + + let mut current_offset = start_offset; + let mut write_buffer = Vec::with_capacity(PAGE_SIZE_USIZE); + + // 2. Build Leaves + // Chunk items into leaf nodes + let mut next_level_pointers: Vec<(K, u64)> = Vec::new(); + + for chunk in items.chunks(self.leaf_order) { + let mut node = BPlusTreeNode::::new(true); + for (k, v) in chunk { + node.keys.push(k.clone()); + node.value_info.push(v.clone()); + } + + let node_offset = current_offset; + current_offset = node.serialize_to_block(file, &mut write_buffer, node_offset)?; + + if !node.keys.is_empty() { + next_level_pointers.push((node.keys[0].clone(), node_offset)); + } + } + + // 3. Build Internal Levels recursively + while next_level_pointers.len() > 1 { + let mut parent_level_pointers: Vec<(K, u64)> = Vec::new(); + let children = next_level_pointers; // Move + + // Chunk size: `inner_order + 1` (pointers). + for chunk in children.chunks(self.inner_order + 1) { + let mut node = BPlusTreeNode::::new(false); + let mut pointers = Vec::new(); + + // First pointer + if let Some((_, off)) = chunk.first() { + pointers.push(*off); + } + + // Subsequent pointers and separators + for (k, off) in &chunk[1..] { + node.keys.push(k.clone()); + pointers.push(*off); + } + + let node_offset = current_offset; + + // Use serialize_internal_with_offsets + current_offset = node.serialize_internal_with_offsets(file, &mut write_buffer, node_offset, &pointers)?; + + // Track for upper level + if let Some((k, _)) = chunk.first() { + parent_level_pointers.push((k.clone(), node_offset)); + } + } + next_level_pointers = parent_level_pointers; + } + + // 4. Return Root Offset + if let Some((_, root_off)) = next_level_pointers.first() { + Ok(*root_off) + } else { + Ok(start_offset) + } + } + pub fn load(filepath: &Path) -> io::Result { let mut file = File::open(filepath)?; @@ -1643,6 +1748,50 @@ where } } +fn count_items( + file: &mut R, + buffer: &mut Vec, + cache: &mut IndexMap>, + start_offset: u64, +) -> Result +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut count = 0; + let mut stack = vec![start_offset]; + while let Some(offset) = stack.pop() { + let (node, pointers) = if let Some(cached_block) = cache.get_mut(&offset) { + let data = cached_block.clone(); + let temp_buffer = data; + let res = BPlusTreeNode::::deserialize_from_block_slice(&temp_buffer, file, false) + .map_err(BPlusTreeError::from)?; + if let Some(removed) = cache.shift_remove(&offset) { + cache.insert(offset, removed); + } + res + } else { + match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { + Ok((node, pointers)) => { + if cache.len() >= CACHE_CAPACITY { + cache.shift_remove_index(0); + } + cache.insert(offset, buffer.to_owned()); + (node, pointers) + } + Err(err) => return Err(BPlusTreeError::from(err)), + } + }; + + if node.is_leaf { + count += node.keys.len(); + } else if let Some(ptrs) = pointers { + stack.extend(ptrs); + } + } + Ok(count) +} + /// `BPlusTreeQuery` performs on-disk queries without loading the entire tree into memory. /// For frequent queries, consider using `BPlusTree::load()` instead, which loads the full tree into memory @@ -1695,6 +1844,20 @@ where query_tree(&mut self.file, &mut self.buffer, &mut self.cache, key, self.root_offset) } + pub fn is_empty(&mut self) -> Result { + let (node, _) = BPlusTreeNode::::deserialize_from_block( + &mut self.file, + &mut self.buffer, + self.root_offset, + false, + )?; + Ok(node.is_leaf && node.keys.is_empty()) + } + + pub fn len(&mut self) -> Result { + count_items::(&mut self.file, &mut self.buffer, &mut self.cache, self.root_offset) + } + pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { query_tree_le(&mut self.file, &mut self.buffer, &mut self.cache, key, self.root_offset) } @@ -2002,97 +2165,94 @@ where query_tree(&mut reader, &mut self.read_buffer, &mut self.cache, key, self.root_offset) } + pub fn is_empty(&mut self) -> Result { + let mut reader = utils::file_reader(&mut self.file); + let (node, _) = BPlusTreeNode::::deserialize_from_block( + &mut reader, + &mut self.read_buffer, + self.root_offset, + false, + )?; + Ok(node.is_leaf && node.keys.is_empty()) + } + + pub fn len(&mut self) -> Result { + let mut reader = utils::file_reader(&mut self.file); + count_items::(&mut reader, &mut self.read_buffer, &mut self.cache, self.root_offset) + } + pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { let mut reader = utils::file_reader(&mut self.file); query_tree_le(&mut reader, &mut self.read_buffer, &mut self.cache, key, self.root_offset) } - fn update_recursive( - &mut self, - offset: u64, - key: &K, - value: &V, - ) -> Result { - let mut reader = utils::file_reader(&mut self.file); - let (mut node, pointers) = BPlusTreeNode::::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false)?; - - if node.is_leaf { - match node.keys.binary_search(key) { - Ok(idx) => { - // COW: Write new value with adaptive LZ4 compression - let raw_bytes = binary_serialize(value)?; - - - - let (flag, payload) = compress_if_beneficial(&raw_bytes); - - self.file.seek(SeekFrom::End(0))?; - let val_offset = self.file.stream_position()?; - - // Write: [flag:1][payload] - self.file.write_all(&[flag])?; - // NOTE: For LZ4, payload already includes original length (prepended) - self.file.write_all(&payload)?; - - // stored_len includes flag + payload - let stored_len = 1 + payload.len(); - - // Update leaf metadata - node.value_info[idx] = ValueInfo { - mode: ValueStorageMode::Single(val_offset), - length: u32::try_from(stored_len).map_err(to_io_error)?, - compressed_cache: None, - }; - - // Write new leaf node at end of file - self.file.seek(SeekFrom::End(0))?; - let new_leaf_offset = self.file.stream_position()?; - node.serialize_to_block(&mut self.file, &mut self.write_buffer, new_leaf_offset)?; - Ok(new_leaf_offset) - } - Err(_) => Err(BPlusTreeError::KeyNotFound), - } - } else { - let child_idx = get_entry_index_upper_bound::(&node.keys, key); - if let Some(mut pters) = pointers { - if let Some(&child_offset) = pters.get(child_idx) { - // Recurse to get a new child offset - let new_child_offset = self.update_recursive(child_offset, key, value)?; - - // Update current node's pointers - pters[child_idx] = new_child_offset; - - // COW: Write new internal node at end of file - self.file.seek(SeekFrom::End(0))?; - let new_node_offset = self.file.stream_position()?; - - // Use the robust helper to serialize the internal node with updated child pointers - node.serialize_internal_with_offsets(&mut self.file, &mut self.write_buffer, new_node_offset, &pters)?; - Ok(new_node_offset) - } else { - Err(BPlusTreeError::InvalidStructure("Child pointer not found in internal node".into())) - } - } else { - Err(BPlusTreeError::InvalidStructure("Internal node missing pointers invariant violation".into())) - } - } - } - - pub fn update(&mut self, key: &K, value: V) -> Result { - let new_root_offset = self.update_recursive(self.root_offset, key, &value)?; + let refs = [(key, &value)]; + let new_root_offset = self.update_batch_recursive(self.root_offset, &refs)?; // Atomic Header Swap - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; - self.file.write_all(&new_root_offset.to_le_bytes())?; - self.file.flush()?; - self.file.sync_all()?; + self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; + self.file.write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; + self.file.flush().map_err(BPlusTreeError::Io)?; + self.file.sync_all().map_err(BPlusTreeError::Io)?; self.root_offset = new_root_offset; Ok(new_root_offset) } + /// Update multiple items in batch. This is more efficient than calling `update()` multiple times + /// as it performs all updates and then commits the final root offset once. + /// returns The final root offset after all updates, or an error if any update fails + fn update_batch_recursive( + &mut self, + offset: u64, + items: &[(&K, &V)], + ) -> Result { + let mut reader = utils::file_reader(&mut self.file); + let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false)?; + + if node.is_leaf { + for (key, value) in items { + match node.keys.binary_search(key) { + Ok(idx) => { + let (val_off, val_len) = self.insert_value_to_disk(value).map_err(BPlusTreeError::Io)?; + node.value_info[idx] = ValueInfo { + mode: ValueStorageMode::Single(val_off), + length: val_len, + compressed_cache: None, + }; + } + Err(_) => return Err(BPlusTreeError::KeyNotFound), + } + } + let new_offset = self.write_node(&node).map_err(BPlusTreeError::Io)?; + Ok(new_offset) + } else { + let mut pointers = pointers_opt.ok_or_else(|| BPlusTreeError::InvalidStructure("Internal node missing pointers".into()))?; + + let mut current_idx = 0; + while current_idx < items.len() { + let first_key_in_group = items[current_idx].0; + let child_idx = get_entry_index_upper_bound::(&node.keys, first_key_in_group); + + let mut group_end = current_idx + 1; + while group_end < items.len() && get_entry_index_upper_bound::(&node.keys, items[group_end].0) == child_idx { + group_end += 1; + } + + let sub_items = &items[current_idx..group_end]; + let new_child_offset = self.update_batch_recursive(pointers[child_idx], sub_items)?; + pointers[child_idx] = new_child_offset; + + current_idx = group_end; + } + + let new_offset = self.write_internal_node(&node, &pointers).map_err(BPlusTreeError::Io)?; + Ok(new_offset) + } + } + /// Update multiple items in batch. This is more efficient than calling `update()` multiple times /// as it performs all updates and then commits the final root offset once. /// returns The final root offset after all updates, or an error if any update fails @@ -2101,21 +2261,19 @@ where return Ok(self.root_offset); } - let mut current_root = self.root_offset; + let mut sorted_items = items.to_vec(); + sorted_items.sort_by(|a, b| a.0.cmp(b.0)); - // Perform all updates sequentially - for (key, value) in items { - current_root = self.update_recursive(current_root, key, value)?; - } + let new_root_offset = self.update_batch_recursive(self.root_offset, &sorted_items)?; // Atomic Header Swap - only once at the end - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; - self.file.write_all(¤t_root.to_le_bytes())?; - self.file.flush()?; - self.file.sync_all()?; + self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; + self.file.write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; + self.file.flush().map_err(BPlusTreeError::Io)?; + self.file.sync_all().map_err(BPlusTreeError::Io)?; - self.root_offset = current_root; - Ok(current_root) + self.root_offset = new_root_offset; + Ok(new_root_offset) } @@ -2156,104 +2314,139 @@ where Ok(offset) } - fn upsert_recursive(&mut self, offset: u64, key: K, value: V) -> io::Result<(u64, Option<(K, u64)>)> { + + fn upsert_batch_recursive( + &mut self, + offset: u64, + items: &[(&K, &V)], + ) -> io::Result<(u64, Vec<(K, u64)>)> { + let mut reader = utils::file_reader(&mut self.file); let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block( - &mut self.file, + &mut reader, &mut self.read_buffer, offset, false, // shallow )?; if node.is_leaf { - match node.keys.binary_search(&key) { - Ok(idx) => { - let (val_off, val_len) = self.insert_value_to_disk(&value)?; - let new_info = ValueInfo { mode: ValueStorageMode::Single(val_off), length: val_len, compressed_cache: None }; - node.value_info[idx] = new_info; + for (key, value) in items { + let (val_off, val_len) = self.insert_value_to_disk(value)?; + let new_info = ValueInfo { mode: ValueStorageMode::Single(val_off), length: val_len, compressed_cache: None }; - let new_offset = self.write_node(&node)?; - Ok((new_offset, None)) - } - Err(idx) => { - let (val_off, val_len) = self.insert_value_to_disk(&value)?; - let new_info = ValueInfo { mode: ValueStorageMode::Single(val_off), length: val_len, compressed_cache: None }; - node.keys.insert(idx, key); - node.value_info.insert(idx, new_info); - - if node.keys.len() > self.leaf_order { - let median = self.leaf_order >> 1; - let mut right_node = BPlusTreeNode::new(true); - right_node.keys = node.keys.split_off(median); - right_node.value_info = node.value_info.split_off(median); - - let promoted_key = right_node.keys.first().ok_or_else(|| io::Error::other("Split resulted in empty right node keys"))?.clone(); - - let left_offset = self.write_node(&node)?; - let right_offset = self.write_node(&right_node)?; - - Ok((left_offset, Some((promoted_key, right_offset)))) - } else { - let new_offset = self.write_node(&node)?; - Ok((new_offset, None)) + match node.keys.binary_search(key) { + Ok(idx) => { + node.value_info[idx] = new_info; + } + Err(idx) => { + node.keys.insert(idx, (*key).clone()); + node.value_info.insert(idx, new_info); } } } - } else { - let mut pointers = pointers_opt.unwrap(); - let idx = get_entry_index_upper_bound(&node.keys, &key); - let child_offset = *pointers.get(idx).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Pointer index out of bounds"))?; - let (new_child_offset, split_res) = self.upsert_recursive(child_offset, key, value)?; - pointers[idx] = new_child_offset; + let mut leaf_promotions = Vec::new(); + while node.keys.len() > self.leaf_order { + let median_idx = node.keys.len() / 2; + let mut right_node = BPlusTreeNode::new(true); + right_node.keys = node.keys.split_off(median_idx); + right_node.value_info = node.value_info.split_off(median_idx); - if let Some((median_key, right_child_offset)) = split_res { - node.keys.insert(idx, median_key); - pointers.insert(idx + 1, right_child_offset); - - if node.keys.len() > self.inner_order { - let median = self.inner_order >> 1; - let mut right_node = BPlusTreeNode::new(false); - let right_pointers = pointers.split_off(median + 1); - right_node.keys = node.keys.split_off(median + 1); - - let promoted_key = node.keys.pop().unwrap(); - - let left_offset = self.write_internal_node(&node, &pointers)?; - let right_offset = self.write_internal_node(&right_node, &right_pointers)?; - - Ok((left_offset, Some((promoted_key, right_offset)))) - } else { - let new_offset = self.write_internal_node(&node, &pointers)?; - Ok((new_offset, None)) - } - } else { - let new_offset = self.write_internal_node(&node, &pointers)?; - Ok((new_offset, None)) + let promoted_key = right_node.keys[0].clone(); + let right_offset = self.write_node(&right_node)?; + leaf_promotions.push((promoted_key, right_offset)); } + + let new_leaf_offset = self.write_node(&node)?; + Ok((new_leaf_offset, leaf_promotions)) + } else { + let mut pointers = pointers_opt.ok_or_else(|| io::Error::other("Internal node missing pointers"))?; + let mut current_idx = 0; + + while current_idx < items.len() { + let first_key_in_group = items[current_idx].0; + let child_idx = get_entry_index_upper_bound::(&node.keys, first_key_in_group); + + let mut group_end = current_idx + 1; + while group_end < items.len() && get_entry_index_upper_bound::(&node.keys, items[group_end].0) == child_idx { + group_end += 1; + } + + let sub_items = &items[current_idx..group_end]; + let (new_child_offset, child_promotions) = self.upsert_batch_recursive(pointers[child_idx], sub_items)?; + pointers[child_idx] = new_child_offset; + + for (median_key, right_child_offset) in child_promotions { + let insert_idx = get_entry_index_upper_bound::(&node.keys, &median_key); + node.keys.insert(insert_idx, median_key); + pointers.insert(insert_idx + 1, right_child_offset); + } + + current_idx = group_end; + } + + let mut node_promotions = Vec::new(); + while node.keys.len() > self.inner_order { + let median_idx = node.keys.len() / 2; + let mut right_node = BPlusTreeNode::new(false); + + let promoted_key = node.keys.remove(median_idx); + right_node.keys = node.keys.split_off(median_idx); + let right_pointers = pointers.split_off(median_idx + 1); + + let right_offset = self.write_internal_node(&right_node, &right_pointers)?; + node_promotions.push((promoted_key, right_offset)); + } + + let new_offset = self.write_internal_node(&node, &pointers)?; + Ok((new_offset, node_promotions)) } } + fn build_higher_levels(&mut self, base_offset: u64, mut promotions: Vec<(K, u64)>) -> io::Result { + if promotions.is_empty() { + return Ok(base_offset); + } + promotions.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut node = BPlusTreeNode::::new(false); + let mut pointers = vec![base_offset]; + for (key, ptr) in promotions { + node.keys.push(key); + pointers.push(ptr); + } + + if node.keys.len() <= self.inner_order { + return self.write_internal_node(&node, &pointers); + } + + let mut next_level_promotions = Vec::new(); + while node.keys.len() > self.inner_order { + let median_idx = node.keys.len() / 2; + let mut right_node = BPlusTreeNode::new(false); + let promoted_key = node.keys.remove(median_idx); + right_node.keys = node.keys.split_off(median_idx); + let right_pointers = pointers.split_off(median_idx + 1); + let right_offset = self.write_internal_node(&right_node, &right_pointers)?; + next_level_promotions.push((promoted_key, right_offset)); + } + let left_offset = self.write_internal_node(&node, &pointers)?; + self.build_higher_levels(left_offset, next_level_promotions) + } + /// Insert or update multiple items in batch (upsert). - /// Uses disk-based recursive insertion (COW) to avoid loading full tree. + /// Uses disk-based recursive traversal for efficiency, processing each node only once. pub fn upsert_batch(&mut self, items: &[(&K, &V)]) -> io::Result { if items.is_empty() { return Ok(self.root_offset); } - let mut current_root = self.root_offset; + let mut sorted_items = items.to_vec(); + sorted_items.sort_by(|a, b| a.0.cmp(b.0)); - for (key, value) in items { - let (new_root, split) = self.upsert_recursive(current_root, (*key).clone(), (*value).clone())?; - current_root = new_root; + let (mut current_root, promotions) = self.upsert_batch_recursive(self.root_offset, &sorted_items)?; - if let Some((median_key, right_child)) = split { - let mut new_root_node = BPlusTreeNode::::new(false); - new_root_node.keys.push(median_key); - let pointers = vec![current_root, right_child]; - - current_root = self.write_internal_node(&new_root_node, &pointers)?; - } - } + // Handle promotions (splits) using balanced approach + current_root = self.build_higher_levels(current_root, promotions)?; self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; self.file.write_all(¤t_root.to_le_bytes())?; @@ -2266,13 +2459,65 @@ where /// Garbage Collection: Compacts the file by rewriting only live blocks sequentially. pub fn compact(&mut self, filepath: &Path) -> io::Result<()> { - // 1. Reload the current tree fully from the live root - let mut tree = BPlusTree::::load(filepath)?; - // 2. Setting dirty=true forces store_internal() to rewrite the file sequentially. - // We use store_internal because we already hold the lock in self._lock. - tree.dirty = true; - let new_root_offset = tree.store_internal(filepath)?; - self.root_offset = new_root_offset; + let mut value_infos: Vec<(K, ValueInfo)> = Vec::new(); + let mut temp_file = NamedTempFile::new_in(filepath.parent().unwrap_or(Path::new(".")))?; + + // 1. Write Header placeholder + temp_file.seek(SeekFrom::Start(0))?; + temp_file.write_all(MAGIC)?; + temp_file.write_all(&STORAGE_VERSION.to_le_bytes())?; + temp_file.seek(SeekFrom::Start(HEADER_SIZE))?; // Skip to start of content + + let mut current_offset = HEADER_SIZE; + + // 2. Iterate source and write values immediately (Streaming) + { + let mut query = BPlusTreeQuery::::try_new(filepath).map_err(to_io_error)?; + let mut write_buffer = std::io::BufWriter::new(&mut temp_file); + + for (k, v) in query.iter() { + let value_bytes = binary_serialize(&v)?; + let len = value_bytes.len(); + + let offset = current_offset; + + // Write value header (flag + payload) + write_buffer.write_all(&[COMPRESSION_FLAG_NONE])?; + write_buffer.write_all(&value_bytes)?; + + current_offset += 1 + len as u64; + + let stored_len = u32::try_from(len + 1).map_err(to_io_error)?; + value_infos.push((k, ValueInfo { + mode: ValueStorageMode::Single(offset), + length: stored_len, + compressed_cache: None, + })); + } + write_buffer.flush()?; + } + + // 3. Build Tree Structure + current_offset = temp_file.seek(SeekFrom::End(0))?; + + let mut tree = BPlusTree::::new(); + let root_offset = tree.bulk_build_from_infos(&mut temp_file, &value_infos, current_offset)?; + + // 4. Update Header + temp_file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; + temp_file.write_all(&root_offset.to_le_bytes())?; + + temp_file.flush()?; + temp_file.as_file().sync_all()?; + + // 5. Atomic Replace + temp_file.persist(filepath).map_err(to_io_error)?; + + // 6. Refresh state + self.root_offset = root_offset; + self.file = utils::open_read_write_file(filepath)?; + self.cache.clear(); + Ok(()) } } @@ -2482,6 +2727,81 @@ mod tests { }); } + #[test] + fn test_upsert_batch() -> io::Result<()> { + let tempdir = tempfile::tempdir()?; + let filepath = tempdir.path().join("upsert_batch_test.bin"); + + // 1. Create initial tree + let mut tree = BPlusTree::::new(); + tree.insert(1, Record { id: 1, data: "original 1".to_string() }); + tree.insert(2, Record { id: 2, data: "original 2".to_string() }); + tree.store(&filepath)?; + + // 2. Open for update and upsert batch + let mut update = BPlusTreeUpdate::::try_new(&filepath)?; + let r1_new = Record { id: 1, data: "updated 1".to_string() }; + let r3_new = Record { id: 3, data: "new 3".to_string() }; + + update.upsert_batch(&[ + (&1, &r1_new), + (&3, &r3_new), + ])?; + + // 3. Verify with query + let mut query = BPlusTreeQuery::::try_new(&filepath)?; + assert_eq!(query.query(&1).unwrap(), Some(r1_new)); + assert_eq!(query.query(&2).unwrap(), Some(Record { id: 2, data: "original 2".to_string() })); + assert_eq!(query.query(&3).unwrap(), Some(r3_new)); + + Ok(()) + } + + #[test] + fn len_test() -> io::Result<()> { + let test_size = 100; + let mut tree = BPlusTree::::new(); + + // Initial state + assert_eq!(tree.len(), 0); + assert!(tree.is_empty()); + + for i in 1..=test_size { + tree.insert(i, Record { + id: i, + data: format!("data {i}"), + }); + assert_eq!(tree.len(), i as usize); + assert!(!tree.is_empty()); + } + + let tempdir = tempfile::tempdir()?; + let filepath = tempdir.path().join("len_test.bin"); + tree.store(&filepath)?; + + // Test BPlusTreeQuery len + let mut query: BPlusTreeQuery = BPlusTreeQuery::try_new(&filepath)?; + assert_eq!(query.len().expect("Query len failed"), test_size as usize); + assert!(!query.is_empty().expect("Query is_empty failed")); + + // Test BPlusTreeUpdate len and modifications + let mut update: BPlusTreeUpdate = BPlusTreeUpdate::try_new(&filepath)?; + assert_eq!(update.len().expect("Update len failed"), test_size as usize); + + // Update existing key - length should stay same + update.update(&1, Record { id: 1, data: "updated".to_string() }).map_err(|e| e.to_io())?; + assert_eq!(update.len().expect("Update len failed after update"), test_size as usize); + + // Insert new key - length should increase + update.upsert_batch(&[ + (&(test_size + 1), &Record { id: test_size + 1, data: "new".to_string() }) + ])?; + assert_eq!(update.len().expect("Update len failed after insert"), (test_size + 1) as usize); + + Ok(()) + } + + #[test] fn iterator_test() -> io::Result<()> { let mut tree = BPlusTree::::new(); @@ -2922,6 +3242,43 @@ mod tests { Ok(()) } + + #[test] + fn compact_reopen_test() -> io::Result<()> { + let tempdir = tempfile::tempdir()?; + let filepath = tempdir.path().join("tree_compact_reopen.bin"); + let mut tree = BPlusTree::::new(); + + // Initial write + tree.insert(1, Record { id: 1, data: "Initial".to_string() }); + tree.store(&filepath)?; + drop(tree); + + let mut tree_update = BPlusTreeUpdate::::try_new(&filepath)?; + + // 1. Write something + let r2 = Record { id: 2, data: "BeforeCompact".to_string() }; + tree_update.upsert_batch(&[(&2, &r2)])?; + + // 2. Compact (this replaces the file) + tree_update.compact(&filepath)?; + + // 3. Write something else + let r3 = Record { id: 3, data: "AfterCompact".to_string() }; + tree_update.upsert_batch(&[(&3, &r3)])?; + + drop(tree_update); + + // Verify all data is present in the NEW file + let mut tree_check = BPlusTreeQuery::::try_new(&filepath)?; + + assert!(tree_check.query(&1).map_err(|e| e.to_io())?.is_some(), "Should have key 1"); + assert!(tree_check.query(&2).map_err(|e| e.to_io())?.is_some(), "Should have key 2"); + assert!(tree_check.query(&3).map_err(|e| e.to_io())?.is_some(), "Should have key 3 - if missing, file handle wasn't updated"); + + Ok(()) + } + #[test] fn upsert_batch_mixed_test() -> io::Result<()> { let tempdir = tempfile::tempdir()?; @@ -3091,7 +3448,7 @@ mod tests { let file_size = std::fs::metadata(&filepath)?.len(); - // Expected size without packing: + // Expected size without packing: // 1000 items * 4096 bytes/block = 4,096,000 bytes (~4MB) // Plus internal nodes let unpacked_size_estimate = count as u64 * super::PAGE_SIZE_USIZE as u64; @@ -3099,9 +3456,9 @@ mod tests { println!("File size with packing: {} bytes", file_size); println!("Estimated unpacked size: {} bytes", unpacked_size_estimate); - // We expect significant savings. + // We expect significant savings. // 1000 items * ~60 bytes / 4096 bytes/block ~= 15 blocks - // Plus tree structure overhead. + // Plus tree structure overhead. // Let's be conservative and say it should be less than 10% of unpacked size. assert!(file_size < unpacked_size_estimate / 10, "Packing should reduce size by at least 90%"); @@ -3193,7 +3550,7 @@ mod tests { let mut tree_update = BPlusTreeUpdate::::try_new(&filepath)?; - // Insert 5000 items. + // Insert 5000 items. // 5000 items ensures at least Root -> Internal -> Leaf split (Height 2 or 3). let count = 5000; @@ -3268,10 +3625,14 @@ mod tests { for i in 0..count { updates.push((i, val.clone())); } - let refs: Vec<(&u32, &String)> = updates.iter().map(|(k, v)| (k, v)).collect(); - tree_update.upsert_batch(&refs)?; + // Use individual upsert calls to create fragmentation (one path write per update) + for (k, v) in updates { + let refs = [(&k, &v)]; + tree_update.upsert_batch(&refs)?; + } let size_before = std::fs::metadata(&filepath)?.len(); + // Each individual update writes a full path, increasing file size significantly. assert!(size_before > count as u64 * 4000); // Now Compact @@ -3475,9 +3836,10 @@ mod page_tests { match res { Ok(None) => { assert_eq!(page.header.cell_count, 1); // Original page untouched - }, + } Ok(Some(_)) => panic!("Split of single item should result in None"), Err(e) => panic!("Split of single item should result in no-op, not error: {:?}", e), } } } + diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index 2a0c85c33..4a7bc91e1 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -19,14 +19,14 @@ const XML_PREAMBLE: &str = r#" // // // XML Header via events (DO NOT USE, kept for documentation): // writer.write_event_async(quick_xml::events::Event::Decl(quick_xml::events::BytesDecl::new("1.0", Some("utf-8"), None))) -// .await.map_err(|e| notify_err!(format!("failed to write XML header: {}", e)))?; +// .await.map_err(|e| notify_err!("failed to write XML header: {}", e))?; // // // DOCTYPE via events (DO NOT USE): // writer.write_event_async(quick_xml::events::Event::DocType(quick_xml::events::BytesText::new(r#"tv SYSTEM "xmltv.dtd""#))) -// .await.map_err(|e| notify_err!(format!("failed to write doctype: {}", e)))?; +// .await.map_err(|e| notify_err!("failed to write doctype: {}", e))?; pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playlist: Option<&[PlaylistGroup]>) -> Result<(), TuliproxError> { let file = tokio::fs::File::create(path).await - .map_err(|e| notify_err!(format!("failed to create epg file {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to create epg file {}: {}", path.display(), e))?; // Use a larger buffer for sequential writes to reduce syscalls let mut buf_writer = async_file_writer(file); @@ -35,7 +35,7 @@ pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playl buf_writer .write_all(XML_PREAMBLE.as_bytes()) .await - .map_err(|e| notify_err!(format!("failed to write XML preamble {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to write XML preamble {}: {}", path.display(), e))?; // Build a temporary rename map with zero allocations (uses references) let mut rename_map: HashMap<&str, &str> = HashMap::new(); @@ -56,16 +56,16 @@ pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playl epg .write_to_async(&mut writer, if rename_map.is_empty() { None } else { Some(&rename_map) }) .await - .map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to write epg {}: {}", path.display(), e))?; // Ensure buffers are flushed to the OS and capture any I/O error let mut buf_writer = writer.into_inner(); buf_writer .flush() .await - .map_err(|e| notify_err!(format!("failed to flush epg {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to flush epg {}: {}", path.display(), e))?; - buf_writer.shutdown().await.map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?; + buf_writer.shutdown().await.map_err(|e| notify_err!("failed to write epg {}: {}", path.display(), e))?; debug_if_enabled!("Epg for target {} written to {}", target.name, path.display()); Ok(()) @@ -81,7 +81,7 @@ pub async fn epg_write(cfg: &Config, target: &ConfigTarget, target_path: &Path, debug_if_enabled!("writing xtream epg to {}", epg_path.display()); epg_write_file(target, epg_data, &epg_path, playlist).await?; } - None => return Err(notify_err!(format!("failed to serialize epg for target: {}, storage path not found", target.name))), + None => return Err(notify_err!("failed to write epg for target: {}, storage path not found", target.name)), } } TargetOutput::M3u(_) => { diff --git a/backend/src/repository/library_repository.rs b/backend/src/repository/library_repository.rs new file mode 100644 index 000000000..71f233213 --- /dev/null +++ b/backend/src/repository/library_repository.rs @@ -0,0 +1,54 @@ +use crate::model::AppConfig; +use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; +use shared::error::{TuliproxError, notify_err_res}; +use shared::model::{PlaylistGroup, PlaylistItem, UUIDType, XtreamPlaylistItem}; +use std::path::Path; +use std::sync::Arc; +use indexmap::IndexMap; + +pub async fn persist_input_library_playlist(app_config: &Arc, library_path: &Path, playlist: &[PlaylistGroup]) -> Result<(), TuliproxError> { + if playlist.is_empty() { + return Ok(()); + } + let _file_lock = app_config.file_locks.write_lock(library_path).await; + let mut tree = BPlusTree::new(); + for pg in playlist { + for item in &pg.channels { + let xtream = XtreamPlaylistItem::from(item); + tree.insert(item.header.uuid, xtream); + } + } + match tree.store(library_path) { + Ok(_) => Ok(()), + Err(err) => notify_err_res!("failed to write local library playlist: {} - {err}", library_path.display()) + } +} + + +pub async fn load_input_local_library_playlist(app_config: &Arc, lib_path: &Path) -> Result, TuliproxError> { + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); + + if tokio::fs::try_exists(lib_path).await.unwrap_or(false) { + // Load Items + let _file_lock = app_config.file_locks.read_lock(lib_path).await; + if let Ok(mut query) = BPlusTreeQuery::::try_new(lib_path) { + let mut group_cnt = 0; + for (_, ref item) in query.iter() { + let cat_id = item.group.clone(); + groups.entry(cat_id) + .or_insert_with(|| { + group_cnt += 1; + PlaylistGroup { + id: group_cnt, + title: item.group.clone(), + channels: Vec::new(), + xtream_cluster: item.xtream_cluster, + } + }) + .channels.push(PlaylistItem::from(item)); + } + } + } + + Ok(groups.into_values().collect()) +} diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index 6d63027c5..d47865856 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -37,7 +37,7 @@ impl M3uPlaylistIterator { // TODO use playlist memory cache, but be aware of sorting ! - let m3u_output = target.get_m3u_output().ok_or_else(|| info_err!(format!("Unexpected failure, missing m3u target output for target {}", target.name)))?; + let m3u_output = target.get_m3u_output().ok_or_else(|| info_err!("Unexpected failure, missing m3u target output for target {}", target.name))?; let config = cfg.config.load(); let target_path = ensure_target_storage_path(&config, target.name.as_str())?; let m3u_path = m3u_get_file_path(&target_path); @@ -45,7 +45,7 @@ impl M3uPlaylistIterator { let file_lock = cfg.file_locks.read_lock(&m3u_path).await; let query = BPlusTreeQuery::::try_new(&m3u_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {m3u_path:?} - {err}")))?; + .map_err(|err| info_err!("Could not open BPlusTreeQuery {m3u_path:?} - {err}"))?; let reader = query.disk_iter(); let filter = user_get_bouquet_filter(&config, &user.username, None, TargetType::M3u, XtreamCluster::Live).await; @@ -116,14 +116,14 @@ impl M3uPlaylistIterator { let entry = if let Some(set) = &self.filter { if let Some((current_item, _)) = self.lookup_item.take() { // Avoid cloning strings while filtering - let next_valid = self.reader.find(|(_, pli)| set.contains(pli.group.as_str())); + let next_valid = self.reader.find(|(_, pli)| set.contains(&*pli.group)); self.lookup_item = next_valid.map(|(_k, v)| (v, true)); // has_next handled by iterator usually, but here we just need the item let has_next = self.lookup_item.is_some(); Some((current_item, has_next)) } else { - let current_item = self.reader.find(|(_, item)| set.contains(item.group.as_str())); + let current_item = self.reader.find(|(_, item)| set.contains(&*item.group)); if let Some((_, item)) = current_item { - self.lookup_item = self.reader.find(|(_, item)| set.contains(item.group.as_str())).map(|(_, item)| (item, true)); + self.lookup_item = self.reader.find(|(_, item)| set.contains(&*item.group)).map(|(_, item)| (item, true)); let has_next = self.lookup_item.is_some(); Some((item, has_next)) } else { diff --git a/backend/src/repository/m3u_repository.rs b/backend/src/repository/m3u_repository.rs index 06b244153..d3fc925d9 100644 --- a/backend/src/repository/m3u_repository.rs +++ b/backend/src/repository/m3u_repository.rs @@ -9,8 +9,7 @@ use crate::utils; use crate::utils::{async_file_writer, IO_BUFFER_SIZE}; use indexmap::IndexMap; use log::error; -use shared::error::{create_tuliprox_error, info_err, string_to_io_error}; -use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind}; +use shared::error::{notify_err, info_err, string_to_io_error, str_to_io_error, TuliproxError}; use shared::model::{M3uPlaylistItem, PlaylistGroup}; use shared::model::{PlaylistItem, PlaylistItemType, XtreamCluster}; use std::io::Error; @@ -22,7 +21,7 @@ use tokio::task; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.display() ,$err) + notify_err!("failed to write m3u playlist: {} - {}", $path.display() ,$err) } } @@ -38,7 +37,7 @@ pub fn m3u_get_epg_file_path(target_path: &Path) -> PathBuf { macro_rules! await_playlist_write { ($expr:expr, $fmt:literal $(, $args:expr)* ) => {{ $expr.await.map_err(|err| { - create_tuliprox_error!(TuliproxErrorKind::Notify, $fmt $(, $args)*, err) + notify_err!($fmt $(, $args)*, err) })? }}; } @@ -116,7 +115,7 @@ pub async fn m3u_write_playlist( Ok(()) }) .await - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {err}", m3u_path.display()))??; + .map_err(|err| notify_err!("failed to write m3u playlist: {} - {err}", m3u_path.display()))??; Ok(()) } @@ -165,7 +164,7 @@ pub async fn iter_raw_m3u_playlist(config: &AppConfig, target: &ConfigTarget) -> } let file_lock = config.file_locks.read_lock(&m3u_path).await; match BPlusTreeQuery::::try_new(&m3u_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {m3u_path:?} - {err}"))) { + .map_err(|err| info_err!("Could not open BPlusTreeQuery {m3u_path:?} - {err}")) { Ok(mut query) => { let items: Vec = query.iter().map(|(_, v)| v).collect(); let len = items.len(); @@ -196,13 +195,13 @@ pub async fn persist_input_m3u_playlist(app_config: &Arc, m3u_path: & Ok(()) }) .await - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {err}", m3u_path.display()))??; + .map_err(|err| notify_err!("failed to write m3u playlist: {} - {err}", m3u_path.display()))??; Ok(()) } pub async fn load_input_m3u_playlist(app_config: &Arc, m3u_path: &Path) -> Result, TuliproxError> { - let mut groups: IndexMap = IndexMap::new(); + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); if tokio::fs::try_exists(m3u_path).await.unwrap_or(false) { // Load Items diff --git a/backend/src/repository/mod.rs b/backend/src/repository/mod.rs index 72ca6ef59..b1f80a820 100644 --- a/backend/src/repository/mod.rs +++ b/backend/src/repository/mod.rs @@ -11,4 +11,8 @@ pub mod xtream_playlist_iterator; pub mod user_repository; pub mod storage_const; mod playlist_scratch; +mod playlist_source; +mod library_repository; + +pub use playlist_source::*; diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 7fe7ad4d1..8a7ae5f33 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -1,7 +1,7 @@ use crate::api::model::{AppState, PlaylistM3uStorage, PlaylistStorage, PlaylistStorageState, PlaylistXtreamStorage}; use crate::model::Epg; use crate::model::{AppConfig, ConfigInput, ConfigTarget, TargetOutput}; -use crate::processing::processor::playlist::apply_filter_to_playlist; +use crate::processing::processor::playlist::{apply_filter_to_playlist, PlaylistProcessingContext}; use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; use crate::repository::epg_repository::epg_write; use crate::repository::m3u_repository::{load_input_m3u_playlist, m3u_get_file_path, m3u_write_playlist, persist_input_m3u_playlist}; @@ -12,15 +12,15 @@ use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord}; use crate::repository::xtream_repository::{load_input_xtream_playlist, persist_input_xtream_playlist, xtream_get_file_path, xtream_get_storage_path, xtream_write_playlist}; use crate::utils; use log::info; -use shared::create_tuliprox_error; -use shared::error::TuliproxError; -use shared::error::{info_err, TuliproxErrorKind}; +use crate::repository::{LocalLibraryDiskPlaylistSource, M3uDiskPlaylistSource, MemoryPlaylistSource, PlaylistSource, XtreamDiskPlaylistSource}; +use shared::error::{info_err, TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{InputType, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{is_dash_url, is_hls_url}; +use shared::utils::{is_dash_url, is_hls_url, StringInterner}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::repository::library_repository::{load_input_local_library_playlist, persist_input_library_playlist}; struct LocalEpisodeKey { path: String, @@ -33,7 +33,8 @@ pub struct ProviderEpisodeKey { } pub async fn persist_playlist(app_config: &Arc, playlist: &mut [PlaylistGroup], epg: Option<&Epg>, - target: &ConfigTarget, playlist_state: Option<&Arc>) -> Result<(), Vec> { + target: &ConfigTarget, playlist_state: Option<&Arc>, + interner: &mut StringInterner) -> Result<(), Vec> { let mut errors = vec![]; let config = &app_config.config.load(); let target_path = match ensure_target_storage_path(config, &target.name) { @@ -80,7 +81,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl rewrite_series_info_episode_virtual_id(playlist, &local_library_series, &provider_series); for output in &target.output { - let mut filtered = match output { + let mut filtered: Option> = match output { TargetOutput::Xtream(out) => out.filter.as_ref().and_then(|flt| apply_filter_to_playlist(playlist, flt)), TargetOutput::M3u(out) => out.filter.as_ref().and_then(|flt| apply_filter_to_playlist(playlist, flt)), TargetOutput::Strm(out) => out.filter.as_ref().and_then(|flt| apply_filter_to_playlist(playlist, flt)), @@ -94,7 +95,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl }; let result = match output { - TargetOutput::Xtream(_xtream_output) => xtream_write_playlist(app_config, target, pl).await, + TargetOutput::Xtream(_xtream_output) => xtream_write_playlist(app_config, target, pl, interner).await, TargetOutput::M3u(m3u_output) => m3u_write_playlist(app_config, target, m3u_output, &target_path, pl).await, TargetOutput::Strm(strm_output) => write_strm_playlist(app_config, target, strm_output, pl).await, TargetOutput::HdHomeRun(_hdhomerun_output) => Ok(()), @@ -114,7 +115,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl } if let Err(err) = target_id_mapping.persist() { - errors.push(info_err!(err.to_string())); + errors.push(info_err!("{err}")); } drop(file_lock); @@ -244,7 +245,7 @@ async fn load_target_id_mapping_as_tree(app_config: &AppConfig, target_path: &Pa let target_id_mapping_file = get_target_id_mapping_file(target_path); let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; BPlusTree::::load(&target_id_mapping_file).map_err(|err| - create_tuliprox_error!(TuliproxErrorKind::Info, "Could not find path for target {} err:{err}", &target.name)) + info_err!("Could not find path for target {} err:{err}", &target.name)) } async fn load_xtream_playlist_as_tree(app_config: &AppConfig, storage_path: &Path, cluster: XtreamCluster) -> BPlusTree { @@ -262,15 +263,10 @@ async fn load_xtream_playlist_as_tree(app_config: &AppConfig, storage_path: &Pat async fn load_xtream_target_storage(app_config: &AppConfig, target: &ConfigTarget) -> Result { let config = app_config.config.load(); let target_path = get_target_storage_path(&config, target.name.as_str()).ok_or_else(|| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {}", &target.name - ))?; + info_err!("Could not find path for target {}", &target.name))?; let storage_path = xtream_get_storage_path(&config, target.name.as_str()).ok_or_else(|| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {} xtream output", &target.name))?; + info_err!("Could not find path for target {} xtream output", &target.name))?; let target_id_mapping = load_target_id_mapping_as_tree(app_config, &target_path, target).await?; let live_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Live).await; @@ -288,10 +284,7 @@ async fn load_xtream_target_storage(app_config: &AppConfig, target: &ConfigTarge async fn load_m3u_target_storage(app_config: &AppConfig, target: &ConfigTarget) -> Result { let config = app_config.config.load(); let target_path = get_target_storage_path(&config, target.name.as_str()).ok_or_else(|| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {}", &target.name - ))?; + info_err!("Could not find path for target {}", &target.name))?; let m3u_path = m3u_get_file_path(&target_path); let _file_lock = app_config.file_locks.read_lock(&m3u_path).await; @@ -344,57 +337,99 @@ pub async fn persist_input_playlist(app_config: &Arc, input: &ConfigI let storage_path = match get_input_storage_path(&input.name, working_dir) { Ok(storage_path) => storage_path, Err(err) => { - return (playlist, Some(create_tuliprox_error!( TuliproxErrorKind::Info, "Error creating input storage directory for input '{}' failed: {err}", input.name))); + return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); } }; persist_input_xtream_playlist(app_config, &storage_path, playlist).await } - _ => { - // Persist M3U/Other types + InputType::M3u | InputType::M3uBatch => { + // Persist M3U let working_dir = &app_config.config.load().working_dir; let storage_path = match get_input_storage_path(&input.name, working_dir) { Ok(storage_path) => storage_path, Err(err) => { - return (playlist, Some(create_tuliprox_error!( TuliproxErrorKind::Info, "Error creating input storage directory for input '{}' failed: {err}", input.name))); + return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); } }; - let file_path = get_input_playlist_file_path(&storage_path, input.name.as_str()); + let file_path = get_input_m3u_playlist_file_path(&storage_path, input.name.as_str()); if let Err(err) = persist_input_m3u_playlist(app_config, &file_path, &playlist).await { return (playlist, Some(err)); } (playlist, None) + }, + InputType::Library => { + // Persist local library playlist + let working_dir = &app_config.config.load().working_dir; + let storage_path = match get_input_storage_path(&input.name, working_dir) { + Ok(storage_path) => storage_path, + Err(err) => { + return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); + } + }; + let file_path = get_input_local_library_playlist_file_path(&storage_path, input.name.as_str()); + if let Err(err) = persist_input_library_playlist(app_config, &file_path, &playlist).await { + return (playlist, Some(err)); + } + (playlist, None) } } } -pub async fn load_input_playlist(app_config: &Arc, input: &ConfigInput, clusters: Option<&[XtreamCluster]>) -> Result, TuliproxError> { +pub async fn load_input_playlist(ctx: &PlaylistProcessingContext, input: &ConfigInput, clusters: Option<&[XtreamCluster]>) -> Result, TuliproxError> { + let app_config = &ctx.config; let working_dir = &app_config.config.load().working_dir; let storage_path = get_input_storage_path(&input.name, working_dir) - .map_err(|e| create_tuliprox_error!(TuliproxErrorKind::Info, "Error getting input path: {e}"))?; + .map_err(|e| info_err!("Error getting input path: {e}"))?; + + let disk_based_processing = app_config.config.load().disk_based_processing; match input.input_type { InputType::Xtream | InputType::XtreamBatch => { - let clusters_to_load = if let Some(c) = clusters { - c + if disk_based_processing { + Ok(Box::new(XtreamDiskPlaylistSource::new(app_config, &storage_path).await)) } else { - &XTREAM_CLUSTER - }; - - load_input_xtream_playlist(app_config, &storage_path, clusters_to_load).await + let clusters_to_load = if let Some(c) = clusters { + c + } else { + &XTREAM_CLUSTER + }; + let groups = load_input_xtream_playlist(app_config, &storage_path, clusters_to_load).await?; + Ok(Box::new(MemoryPlaylistSource::new(groups))) + } } - _ => { + InputType::M3u | InputType::M3uBatch => { // Load M3U - let file_path = get_input_playlist_file_path(&storage_path, input.name.as_str()); - load_input_m3u_playlist(app_config, &file_path).await + let file_path = get_input_m3u_playlist_file_path(&storage_path, input.name.as_str()); + if disk_based_processing && file_path.exists() { + Ok(Box::new(M3uDiskPlaylistSource::new(app_config, &file_path).await)) + } else { + let groups = load_input_m3u_playlist(app_config, &file_path).await?; + Ok(Box::new(MemoryPlaylistSource::new(groups))) + } + } + InputType::Library => { + let file_path = get_input_local_library_playlist_file_path(&storage_path, input.name.as_str()); + if disk_based_processing && file_path.exists() { + Ok(Box::new(LocalLibraryDiskPlaylistSource::new(app_config, &file_path).await)) + } else { + let groups = load_input_local_library_playlist(app_config, &file_path).await?; + Ok(Box::new(MemoryPlaylistSource::new(groups))) + } } } } - -fn get_input_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { +fn get_input_m3u_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { let sanitized_input_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); - storage_path.join(format!("{sanitized_input_name}_playlist.{FILE_SUFFIX_DB}")) + storage_path.join(format!("m3u_{sanitized_input_name}.{FILE_SUFFIX_DB}")) +} + +fn get_input_local_library_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { + let sanitized_input_name: String = input_name.chars() + .map(|c| if c.is_alphanumeric() { c } else { '_' }) + .collect(); + storage_path.join(format!("lib_{sanitized_input_name}.{FILE_SUFFIX_DB}")) } \ No newline at end of file diff --git a/backend/src/repository/playlist_source.rs b/backend/src/repository/playlist_source.rs new file mode 100644 index 000000000..c16be6390 --- /dev/null +++ b/backend/src/repository/playlist_source.rs @@ -0,0 +1,525 @@ +use crate::model::AppConfig; +use crate::repository::bplustree::BPlusTreeQuery; +use crate::repository::xtream_repository::xtream_get_file_path; +use crate::utils::FileReadGuard; +use futures::future::BoxFuture; +use indexmap::IndexMap; +use log::{error, warn}; +use serde::{Deserialize, Serialize}; +use shared::model::{M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, UUIDType, XtreamCluster, XtreamPlaylistItem}; +use std::borrow::Cow; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub trait PlaylistSource: Send + Sync { + fn is_memory(&self) -> bool; + fn get_channel_count(&mut self) -> usize; + fn get_group_count(&mut self) -> usize; + fn is_empty(&mut self) -> bool; + #[allow(clippy::wrong_self_convention)] + fn into_items(&mut self) -> Box + Send + '_>; + fn items_mut(&mut self) -> Box + Send + '_>; + fn items<'a>(&'a mut self) -> Box> + Send + 'a>; + fn update_playlist<'a>(&'a mut self, plg: &'a PlaylistGroup) -> BoxFuture<'a, ()>; + fn get_missing_vod_info_count(&mut self) -> usize; + fn get_missing_series_info_count(&mut self) -> usize; + fn deduplicate(&mut self, duplicates: &mut HashSet); + fn take_groups(&mut self) -> Vec; + fn clone_box(&self) -> Box; + fn release_resources(&mut self, cluster: XtreamCluster); + fn obtain_resources(&mut self) -> BoxFuture<'_, ()>; + fn sort_by_provider_ordinal(&mut self); +} + +#[derive(Default)] +pub struct EmptyPlaylistSource {} + +impl PlaylistSource for EmptyPlaylistSource { + fn is_memory(&self) -> bool { true } + fn get_channel_count(&mut self) -> usize { 0 } + fn get_group_count(&mut self) -> usize { 0 } + fn is_empty(&mut self) -> bool { true } + fn into_items(&mut self) -> Box + Send + '_> { Box::new(std::iter::empty()) } + fn items_mut(&mut self) -> Box + Send + '_> { Box::new(std::iter::empty()) } + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { Box::new(std::iter::empty()) } + fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { Box::pin(async move {}) } + fn get_missing_vod_info_count(&mut self) -> usize { 0 } + fn get_missing_series_info_count(&mut self) -> usize { 0 } + fn deduplicate(&mut self, _duplicates: &mut HashSet) { /* noop */ } + fn take_groups(&mut self) -> Vec { vec![] } + fn clone_box(&self) -> Box { Box::new(EmptyPlaylistSource::default()) } + fn release_resources(&mut self, _cluster: XtreamCluster) { /* noop */ } + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { Box::pin(async move {}) } + fn sort_by_provider_ordinal(&mut self) { /* noop */ } +} + +pub struct XtreamDiskPlaylistSource { + app_config: Arc, + storage_path: PathBuf, + live: Option<(BPlusTreeQuery, Arc)>, + vod: Option<(BPlusTreeQuery, Arc)>, + series: Option<(BPlusTreeQuery, Arc)>, +} + +impl XtreamDiskPlaylistSource { + pub(crate) async fn new(app_config: &Arc, storage_path: &Path) -> Self { + let mut source = XtreamDiskPlaylistSource { + app_config: Arc::clone(app_config), + storage_path: storage_path.to_path_buf(), + live: None, + vod: None, + series: None, + }; + source.reload().await; + source + } + + async fn reload(&mut self) { + if self.live.is_none() { + let live_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Live); + self.live = load_bplustree_query::(&self.app_config, &live_path).await + .map(|(query, guard)| (query, Arc::new(guard))); + } + if self.vod.is_none() { + let vod_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Video); + self.vod = load_bplustree_query::(&self.app_config, &vod_path).await + .map(|(query, guard)| (query, Arc::new(guard))); + } + + if self.series.is_none() { + let series_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Series); + self.series = load_bplustree_query::(&self.app_config, &series_path).await + .map(|(query, guard)| (query, Arc::new(guard))); + } + } +} + +impl PlaylistSource for XtreamDiskPlaylistSource { + fn is_memory(&self) -> bool { false } + + fn get_channel_count(&mut self) -> usize { + self.live.as_mut().map_or(0usize, |(t, _)| t.len().unwrap_or(0usize)) + + self.vod.as_mut().map_or(0usize, |(t, _)| t.len().unwrap_or(0usize)) + + self.series.as_mut().map_or(0usize, |(t, _)| t.len().unwrap_or(0usize)) + } + + fn get_group_count(&mut self) -> usize { + let mut groups = HashSet::new(); + if let Some((query, _)) = self.live.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + if let Some((query, _)) = self.vod.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + if let Some((query, _)) = self.series.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + groups.len() + } + + fn is_empty(&mut self) -> bool { + self.live.as_mut().is_none_or(|(q, _)| q.is_empty().unwrap_or(true)) + && self.vod.as_mut().is_none_or(|(q, _)| q.is_empty().unwrap_or(true)) + && self.series.as_mut().is_none_or(|(q, _)| q.is_empty().unwrap_or(true)) + } + + fn into_items(&mut self) -> Box + Send + '_> { + let live = self.live.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| PlaylistItem::from(&item)); + let vod = self.vod.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| PlaylistItem::from(&item)); + let series = self.series.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| PlaylistItem::from(&item)); + Box::new(live.chain(vod).chain(series)) + } + + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + let live = self.live.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + let vod = self.vod.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + let series = self.series.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + Box::new(live.chain(vod).chain(series)) + } + + fn items_mut(&mut self) -> Box + Send + '_> { + warn!("Disk-based playlist sources are read-only. Use clone_source() and convert to memory for mutable access."); + Box::new(std::iter::empty()) + } + + fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { + Box::pin(async move { + warn!("update_playlist should not be called for Xtream Disk playlist"); + // // Drop read guards before write lock + // self.live = None; + // self.vod = None; + // self.series = None; + // + // let xtream_path = xtream_get_file_path(&self.storage_path, plg.xtream_cluster); + // { + // let _lock = self.app_config.file_locks.write_lock(&xtream_path).await; + // if let Ok(mut tree) = BPlusTreeUpdate::::try_new(&xtream_path) { + // let xtream_items: Vec = plg.channels.iter().map(XtreamPlaylistItem::from).collect(); + // let batch: Vec<(&u32, &XtreamPlaylistItem)> = xtream_items.iter().map(|item| (&item.virtual_id, item)).collect(); + // let _ = tree.upsert_batch(&batch); + // } + // } + // self.reload().await; + }) + } + + fn get_missing_vod_info_count(&mut self) -> usize { + let mut count = 0; + if let Some((query, _)) = self.vod.as_mut() { + for (_, item) in query.iter() { + if item.item_type == PlaylistItemType::Video && item.provider_id > 0 && !item.has_details() { + count += 1; + } + } + } + count + } + + fn get_missing_series_info_count(&mut self) -> usize { + let mut count = 0; + if let Some((query, _)) = self.series.as_mut() { + for (_, item) in query.iter() { + if item.item_type == PlaylistItemType::SeriesInfo && item.provider_id > 0 && !item.has_details() { + count += 1; + } + } + } + count + } + + fn deduplicate(&mut self, _duplicates: &mut HashSet) { + warn!("Deduplication is not supported for disk based playlist updates"); + } + + fn take_groups(&mut self) -> Vec { + // Build groups on-the-fly using disk iterator (streams one leaf at a time) + let mut groups_map: IndexMap = IndexMap::new(); + let mut iters: Vec<(XtreamCluster, Box + Send>)> = vec![]; + if let Some((q, _)) = self.live.as_mut() { + iters.push((XtreamCluster::Live, Box::new(q.iter().map(|(_, item)| item)))); + } + if let Some((q, _)) = self.vod.as_mut() { + iters.push((XtreamCluster::Video, Box::new(q.iter().map(|(_, item)| item)))); + } + if let Some((q, _)) = self.series.as_mut() { + iters.push((XtreamCluster::Series, Box::new(q.iter().map(|(_, item)| item)))); + } + + for (cluster, iter) in iters { + for item in iter { + groups_map.entry(item.category_id) + .or_insert_with(|| PlaylistGroup { + id: item.category_id, + title: item.group.clone(), + channels: vec![], + xtream_cluster: cluster, + }) + .channels.push(PlaylistItem::from(&item)); + } + } + + // Sort channels within each group + for group in groups_map.values_mut() { + group.channels.sort_by_key(|item| item.header.source_ordinal); + } + + // Sort groups based on the source_ordinal of their first channel + let mut groups: Vec = groups_map.into_values().collect(); + groups.sort_by_key(|group| { + group.channels.first().map_or(u32::MAX, |c| c.header.source_ordinal) + }); + groups + } + + fn clone_box(&self) -> Box { + let live_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Live); + let vod_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Video); + let series_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Series); + + let live = self.live.as_ref().and_then(|(_, guard)| { + BPlusTreeQuery::try_new(&live_path).ok().map(|q| (q, Arc::clone(guard))) + }); + let vod = self.vod.as_ref().and_then(|(_, guard)| { + BPlusTreeQuery::try_new(&vod_path).ok().map(|q| (q, Arc::clone(guard))) + }); + let series = self.series.as_ref().and_then(|(_, guard)| { + BPlusTreeQuery::try_new(&series_path).ok().map(|q| (q, Arc::clone(guard))) + }); + + Box::new(Self { + app_config: Arc::clone(&self.app_config), + storage_path: self.storage_path.clone(), + live, + vod, + series, + }) + } + + fn release_resources(&mut self, cluster: XtreamCluster) { + match cluster { + XtreamCluster::Live => self.live = None, + XtreamCluster::Video => self.vod = None, + XtreamCluster::Series => self.series = None, + } + } + + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { + Box::pin(async move { + self.reload().await; + }) + } + fn sort_by_provider_ordinal(&mut self) { + warn!("Sorting by provider ordinal is not supported for disk based playlists"); + } +} + +macro_rules! impl_single_file_disk_source { + ($name:ident, $key_type:tt, $entry_type:tt) => { + paste::paste! { + pub struct [<$name DiskPlaylistSource>] { + app_config: Arc, + file_path: PathBuf, + playlist: Option>, + guard: Option>, + } + + impl [<$name DiskPlaylistSource>] { + pub(crate) async fn new(app_config: &Arc, file_path: &Path) -> Self { + let mut source = Self { + app_config: Arc::clone(app_config), + file_path: file_path.to_path_buf(), + playlist: None, + guard: None, + }; + source.reload().await; + source + } + + async fn reload(&mut self) { + self.guard = None; + self.playlist = load_bplustree_query::<$key_type, $entry_type>(&self.app_config, &self.file_path).await + .map(|(query, guard)| { + self.guard = Some(Arc::new(guard)); + query + }); + } + } + + impl PlaylistSource for [<$name DiskPlaylistSource>] { + + fn get_channel_count(&mut self) -> usize { self.playlist.as_mut().map_or(0usize, |t: &mut BPlusTreeQuery<$key_type, $entry_type>| t.len().unwrap_or(0usize)) } + + fn is_memory(&self) -> bool { false } + + fn get_group_count(&mut self) -> usize { + let mut groups = HashSet::new(); + if let Some(query) = self.playlist.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + groups.len() + } + + fn is_empty(&mut self) -> bool { self.playlist.as_mut().map_or(true, |t| t.is_empty().unwrap_or(true)) } + + fn into_items(&mut self) -> Box + Send + '_> { + if let Some(q) = self.playlist.as_mut() { + Box::new(q.iter().map(|(_, item)| PlaylistItem::from(&item))) + } else { + Box::new(std::iter::empty()) + } + } + + + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + if let Some(pl) = self.playlist.as_mut() { + let iter = pl.iter().map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + Box::new(iter) + } else { + Box::new(std::iter::empty()) + } + } + + fn items_mut(&mut self) -> Box + Send + '_> { + warn!("Disk-based playlist sources are read-only. Use clone_source() and convert to memory for mutable access."); + Box::new(std::iter::empty()) + } + + fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { + Box::pin(async move { + warn!("update_playlist should not be called for M3U Disk playlist"); + }) + } + + fn get_missing_vod_info_count(&mut self) -> usize { 0 } + fn get_missing_series_info_count(&mut self) -> usize { 0 } + fn deduplicate(&mut self, _duplicates: &mut HashSet) { + warn!("Deduplication is not supported for disk based playlist updates"); + } + fn take_groups(&mut self) -> Vec { + // Build groups on-the-fly using disk iterator (streams one leaf at a time) + if let Some(q) = self.playlist.as_mut() { + let mut groups_map: IndexMap, PlaylistGroup> = IndexMap::new(); + for (_, item) in q.iter() { + groups_map.entry(item.group.clone()) + .or_insert_with(|| PlaylistGroup { + id: 0, + title: item.group.clone(), + channels: vec![], + xtream_cluster: XtreamCluster::try_from(item.item_type).unwrap_or(XtreamCluster::Live), + }) + .channels.push(PlaylistItem::from(&item)); + } + // Sort channels within each group + for group in groups_map.values_mut() { + group.channels.sort_by_key(|item| item.header.source_ordinal); + } + + // Sort groups based on the source_ordinal of their first channel + let mut groups: Vec = groups_map.into_values().collect(); + groups.sort_by_key(|group| { + group.channels.first().map_or(u32::MAX, |c| c.header.source_ordinal) + }); + groups + } else { + vec![] + } + } + fn clone_box(&self) -> Box { + let playlist = if self.playlist.is_some() && self.guard.is_some() { + BPlusTreeQuery::try_new(&self.file_path).ok() + } else { None }; + + Box::new(Self { + app_config: Arc::clone(&self.app_config), + file_path: self.file_path.clone(), + playlist, + guard: self.guard.clone(), + }) + } + + fn release_resources(&mut self, _cluster: XtreamCluster) { + self.guard = None; + self.playlist = None; + } + + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { + Box::pin(async move { + self.reload().await; + }) + } + + fn sort_by_provider_ordinal(&mut self) { + warn!("Sorting by provider ordinal is not supported for disk based playlists"); + } + } + } + }; +} + +impl_single_file_disk_source!(M3u, String, M3uPlaylistItem); + +impl_single_file_disk_source!(LocalLibrary, UUIDType, XtreamPlaylistItem); + +pub struct MemoryPlaylistSource { + playlist: Arc>, +} + +impl MemoryPlaylistSource { + pub(crate) fn new(groups: Vec) -> Self { + Self { playlist: Arc::new(groups) } + } + + pub fn boxed(self) -> Box { + Box::new(self) + } +} + +impl Default for MemoryPlaylistSource { + fn default() -> Self { + Self { playlist: Arc::new(vec![]) } + } +} + +impl PlaylistSource for MemoryPlaylistSource { + fn is_memory(&self) -> bool { true } + fn get_channel_count(&mut self) -> usize { self.playlist.iter().map(|group| group.channels.len()).sum() } + fn get_group_count(&mut self) -> usize { self.playlist.len() } + fn is_empty(&mut self) -> bool { self.playlist.is_empty() } + fn into_items(&mut self) -> Box + Send + '_> { + let playlist = Arc::make_mut(&mut self.playlist); + Box::new(playlist.iter_mut().flat_map(|group| group.channels.drain(..))) + } + fn items_mut(&mut self) -> Box + Send + '_> { + let playlist = Arc::make_mut(&mut self.playlist); + Box::new(playlist.iter_mut().flat_map(|group| group.channels.iter_mut())) + } + + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + Box::new( + self.playlist.iter() + .flat_map(|group| group.channels.iter().map(Cow::Borrowed)) + ) + } + + fn update_playlist<'a>(&'a mut self, plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { + Box::pin(async move { + let playlist = Arc::make_mut(&mut self.playlist); + for grp in playlist.iter_mut() { + if grp.id == plg.id { + grp.channels.extend(plg.channels.iter().cloned()); + return; + } + } + playlist.push(plg.clone()); + }) + } + fn get_missing_vod_info_count(&mut self) -> usize { + self.playlist.iter() + .flat_map(|plg| &plg.channels) + .filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video + && pli.header.item_type == PlaylistItemType::Video + && !pli.has_details()).count() + } + fn get_missing_series_info_count(&mut self) -> usize { + self.playlist.iter() + .flat_map(|plg| &plg.channels) + .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Series + && pli.header.item_type == PlaylistItemType::SeriesInfo + && pli.get_provider_id().is_some_and(|id| id > 0) + && !pli.has_details()).count() + } + fn deduplicate(&mut self, duplicates: &mut HashSet) { + let playlist = Arc::make_mut(&mut self.playlist); + for group in playlist { + group.channels.retain(|item| duplicates.insert(item.get_uuid())); + } + } + fn take_groups(&mut self) -> Vec { + std::mem::take(Arc::make_mut(&mut self.playlist)) + } + fn clone_box(&self) -> Box { + Box::new(MemoryPlaylistSource { playlist: Arc::clone(&self.playlist) }) + } + fn release_resources(&mut self, _cluster: XtreamCluster) { /* noop */ } + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { Box::pin(async move {}) } + + fn sort_by_provider_ordinal(&mut self) { + let playlist = Arc::make_mut(&mut self.playlist); + for group in &mut *playlist { + group.channels.sort_by_key(|item| item.header.source_ordinal); + } + playlist.sort_by_key(|group| + group.channels.first().map_or(u32::MAX, |c| c.header.source_ordinal) + ); + } +} + +async fn load_bplustree_query(app_config: &Arc, file_path: &Path) -> Option<(BPlusTreeQuery, FileReadGuard)> +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + P: Serialize + for<'de> Deserialize<'de> + Clone + Send + 'static, +{ + if file_path.exists() { + let guard = app_config.file_locks.read_lock(file_path).await; + match BPlusTreeQuery::::try_new(file_path) { + Ok(query) => Some((query, guard)), + Err(err) => { + error!("Error loading disk playlist {}: {err}", file_path.display()); + None + } + } + } else { None } +} \ No newline at end of file diff --git a/backend/src/repository/storage.rs b/backend/src/repository/storage.rs index 9ead7b5fc..7e242cb06 100644 --- a/backend/src/repository/storage.rs +++ b/backend/src/repository/storage.rs @@ -1,9 +1,10 @@ -use std::path::{Path, PathBuf}; -use shared::error::{TuliproxError}; -use crate::model::{Config}; -use shared::error::{notify_err}; +use crate::model::Config; use crate::repository::storage_const; use crate::utils; +use shared::error::notify_err; +use shared::error::TuliproxError; +use std::path::{Path, PathBuf}; +use shared::notify_err_res; pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { // Join directly with &str to avoid an intermediate PathBuf allocation @@ -14,12 +15,12 @@ pub fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result std::io::R let sanitized_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); - let name = format!("input_{sanitized_name}"); + let name = format!("input_{sanitized_name}"); let path = Path::new(working_dir).join(name); // Create the directory and return the path or propagate the error std::fs::create_dir_all(&path).map(|()| path) } +pub fn ensure_input_storage_path(cfg: &Config, input_name: &str) -> Result { + get_input_storage_path(input_name, &cfg.working_dir) + .map_err(|err| { + notify_err!("Failed to save input data, can't create directory for input {input_name}: {err}") + }) +} + + pub fn get_geoip_path(working_dir: &str) -> PathBuf { Path::new(working_dir).join("geoip.db") } \ No newline at end of file diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 85bf85782..1e9230140 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -5,16 +5,16 @@ use crate::model::{ApiProxyServerInfo, AppConfig, ProxyUserCredentials}; use crate::model::{ConfigTarget, StrmTargetOutput}; use crate::repository::storage::{ensure_target_storage_path}; use crate::repository::storage_const; -use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, IO_BUFFER_SIZE}; +use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, + IO_BUFFER_SIZE}; use chrono::Datelike; use filetime::{set_file_times, FileTime}; use log::{error, trace}; use regex::Regex; use serde::Serialize; -use shared::error::{create_tuliprox_error_result, info_err}; -use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::error::{TuliproxError, info_err_res, info_err}; use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType}; -use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; +use shared::utils::{ arc_str_serde, extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -130,13 +130,15 @@ pub fn strm_get_file_paths(file_prefix: &str, target_path: &Path) -> PathBuf { #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct StrmItemInfo { - group: String, + #[serde(with = "arc_str_serde")] + group: Arc, title: String, item_type: PlaylistItemType, #[serde(default, skip_serializing_if = "Option::is_none")] provider_id: Option, virtual_id: u32, - input_name: String, + #[serde(with = "arc_str_serde")] + input_name: Arc, url: String, #[serde(default, skip_serializing_if = "is_blank_optional_string")] series_name: Option, @@ -219,10 +221,7 @@ async fn prepare_strm_output_directory(path: &Path) -> Result<(), TuliproxError> // Ensure the directory exists if let Err(e) = tokio::fs::create_dir_all(path).await { error!("Failed to create directory {}: {e}", path.display()); - return create_tuliprox_error_result!( - TuliproxErrorKind::Notify, - "Error creating STRM directory: {e}" - ); + return info_err_res!("Error creating STRM directory: {e}"); } Ok(()) } @@ -713,10 +712,7 @@ pub async fn write_strm_playlist( &config.working_dir, Some(std::path::PathBuf::from(&target_output.directory)), ) else { - return Err(info_err!(format!( - "Failed to get file path for {}", - target_output.directory - ))); + return info_err_res!("Failed to get file path for {}",target_output.directory); }; let user_and_server_info = get_credentials_and_server_info(app_config, target_output.username.as_deref()); @@ -801,7 +797,7 @@ pub async fn write_strm_playlist( if failed.is_empty() { Ok(()) } else { - Err(info_err!(failed.join(", "))) + Err(info_err!("{}", failed.join(", "))) } } async fn write_strm_index_file( diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index b5327281f..18fc1db99 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -5,8 +5,7 @@ use crate::repository::user_repository::user_get_bouquet_filter; use crate::repository::xtream_repository::{xtream_get_file_path, xtream_get_storage_path}; use crate::utils::FileReadGuard; use log::error; -use shared::error::info_err; -use shared::error::TuliproxError; +use shared::error::{TuliproxError, info_err, info_err_res}; use shared::model::{PlaylistItemType, TargetType, XtreamCluster, XtreamMappingOptions, XtreamPlaylistItem}; use std::collections::HashSet; @@ -31,17 +30,17 @@ impl XtreamPlaylistIterator { // TODO use playlist memory cache and keep sorted - let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name)))?; + let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected: xtream output required for target {}", target.name))?; let config = app_config.config.load(); if let Some(storage_path) = xtream_get_storage_path(&config, target.name.as_str()) { let xtream_path = xtream_get_file_path(&storage_path, cluster); if !xtream_path.exists() { - return Err(info_err!(format!("No {cluster} entries found for target {}", &target.name))); + return info_err_res!("No {cluster} entries found for target {}", &target.name); } let file_lock = app_config.file_locks.read_lock(&xtream_path).await; let query = BPlusTreeQuery::::try_new(&xtream_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {xtream_path:?} - {err}")))?; + .map_err(|err| info_err!("Could not open BPlusTreeQuery {xtream_path:?} - {err}"))?; let reader = query.disk_iter(); let server_info = app_config.get_user_server_info(user); @@ -67,7 +66,7 @@ impl XtreamPlaylistIterator { lookup_item: None, }) } else { - Err(info_err!(format!("Failed to find xtream storage for target {}", &target.name))) + info_err_res!("Failed to find xtream storage for target {}", &target.name) } } diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index 711602947..6702069e3 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -17,21 +17,20 @@ use indexmap::IndexMap; use log::error; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use shared::error::{create_tuliprox_error, create_tuliprox_error_result, info_err, notify_err, string_to_io_error, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, info_err, notify_err, string_to_io_error, TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemType, SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::get_u32_from_serde_value; -use std::borrow::Cow; +use shared::utils::{arc_str_serde, get_u32_from_serde_value, intern, StringInterner}; use std::collections::HashMap; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use shared::notify_err_res; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_tuliprox_error!( - TuliproxErrorKind::Notify, + notify_err!( "failed to write xtream playlist: {} - {}", $path.display(), $err @@ -40,39 +39,38 @@ macro_rules! cant_write_result { } #[inline] -fn get_collection_path(path: &Path, collection: &str) -> PathBuf { +pub fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } #[inline] -fn get_live_cat_collection_path(path: &Path) -> PathBuf { +pub fn get_live_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_LIVE) } #[inline] -fn get_vod_cat_collection_path(path: &Path) -> PathBuf { +pub fn get_vod_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_VOD) } #[inline] -fn get_series_cat_collection_path(path: &Path) -> PathBuf { +pub fn get_series_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_SERIES) } - -fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { +pub fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { if let Some(path) = xtream_get_storage_path(cfg, target_name) { if std::fs::create_dir_all(&path).is_err() { let msg = format!( "Failed to save xtream data, can't create directory {}", &path.display() ); - return Err(notify_err!(msg)); + return notify_err_res!("{msg}"); } Ok(path) } else { let msg = format!("Failed to save xtream data, can't create directory for target {target_name}"); - Err(notify_err!(msg)) + notify_err_res!("{msg}") } } @@ -167,9 +165,11 @@ fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Optio None } -type CategoryKey = (XtreamCluster, Cow<'static, str>); +pub type CategoryKey = (XtreamCluster, Arc); -async fn load_old_category_ids(path: &Path) -> (u32, HashMap) { +// Because interner is not thread safe we cant use it currently for interning. +// We leave the argument for later optimizations. +async fn load_old_category_ids(path: &Path, _interner: &mut StringInterner) -> (u32, HashMap) { let old_path = path.to_path_buf(); tokio::task::spawn_blocking(move || { let mut result: HashMap = HashMap::new(); @@ -190,7 +190,7 @@ async fn load_old_category_ids(path: &Path) -> (u32, HashMap) if let Some(category_id) = entry.get(crate::model::XC_TAG_CATEGORY_ID).and_then(get_u32_from_serde_value) { if let Value::Object(item) = entry { if let Some(category_name) = get_map_item_as_str(&item, crate::model::XC_TAG_CATEGORY_NAME) { - result.insert((cluster, Cow::Owned(category_name)), category_id); + result.insert((cluster, /*interner.*/intern(&category_name)), category_id); max_id = max_id.max(category_id); } } @@ -226,16 +226,18 @@ pub fn xtream_get_file_path(storage_path: &Path, cluster: XtreamCluster) -> Path } #[derive(Serialize, Deserialize)] -struct CategoryEntry { - category_id: u32, - category_name: String, - parent_id: u32, +pub struct CategoryEntry { + pub category_id: u32, + #[serde(with = "arc_str_serde")] + pub category_name: Arc, + pub parent_id: u32, } pub async fn xtream_write_playlist( app_cfg: &Arc, target: &ConfigTarget, playlist: &mut [PlaylistGroup], + interner: &mut StringInterner, ) -> Result<(), TuliproxError> { let path = { let config = app_cfg.config.load(); @@ -249,7 +251,7 @@ pub async fn xtream_write_playlist( let mut series_col = Vec::with_capacity(50_000); let mut vod_col = Vec::with_capacity(50_000); - let categories = create_categories(playlist, &path).await; + let categories = create_categories(playlist, &path, interner).await; { for (xtream_cluster, category) in categories { match xtream_cluster { @@ -306,25 +308,21 @@ pub async fn xtream_write_playlist( } if !errors.is_empty() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Notify, - "{}", - errors.join("\n") - ); + return info_err_res!("{}", errors.join("\n")); } Ok(()) } -async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<(XtreamCluster, CategoryEntry)> { +async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path, interner: &mut StringInterner) -> Vec<(XtreamCluster, CategoryEntry)> { // preserve category_ids - let (max_cat_id, existing_cat_ids) = load_old_category_ids(path).await; + let (max_cat_id, existing_cat_ids) = load_old_category_ids(path, interner).await; let mut cat_id_counter = max_cat_id; - let mut new_categories: IndexMap<(XtreamCluster, Cow<'_, str>), CategoryEntry> = IndexMap::new(); + let mut new_categories: IndexMap = IndexMap::new(); let mut last_cluster: Option = None; - let mut last_group = String::new(); + let mut last_group = intern(""); let mut last_category_id: u32 = 0; for plg in playlist.iter_mut() { @@ -334,15 +332,15 @@ async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<( for channel in &mut plg.channels { let cluster = channel.header.xtream_cluster; - let group = channel.header.group.as_str(); + let group = &channel.header.group; // Fast path - if last_cluster == Some(cluster) && last_group == group { + if last_cluster == Some(cluster) && &last_group == group { channel.header.category_id = last_category_id; continue; } - let key = (cluster, Cow::Borrowed(group)); + let key = (cluster, Arc::clone(group)); let entry = new_categories.entry(key.clone()).or_insert_with(|| { let cat_id = existing_cat_ids.get(&key).copied().unwrap_or_else(|| { @@ -352,14 +350,13 @@ async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<( CategoryEntry { category_id: cat_id, - category_name: group.to_string(), + category_name: group.clone(), parent_id: 0, } }); last_cluster = Some(cluster); - last_group.clear(); - last_group.push_str(group); + last_group = Arc::clone(group); last_category_id = entry.category_id; channel.header.category_id = last_category_id; @@ -584,7 +581,7 @@ pub async fn iter_raw_xtream_playlist(app_config: &AppConfig, target: &ConfigTar } let file_lock = app_config.file_locks.read_lock(&xtream_path).await; match BPlusTreeQuery::::try_new(&xtream_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {xtream_path:?} - {err}"))) { + .map_err(|err| info_err!("Could not open BPlusTreeQuery {xtream_path:?} - {err}")) { Ok(mut query) => { let items: Vec = query.iter().map(|(_, v)| v).collect(); let len = items.len(); @@ -757,7 +754,7 @@ pub async fn persist_input_xtream_playlist(app_config: &Arc, storage_ let err = if errors.is_empty() { None } else { - Some(create_tuliprox_error!(TuliproxErrorKind::Notify, "{}", errors.join("\n"))) + Some(notify_err!("{}", errors.join("\n"))) }; (result, err) @@ -802,6 +799,41 @@ async fn persist_input_info(app_config: &Arc, storage_path: &Path, cl Ok(()) } +pub async fn persist_input_info_batch(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, + input_name: &str, updates: Vec<(u32, StreamProperties)>) -> Result<(), Error> { + if updates.is_empty() { return Ok(()); } + let xtream_path = xtream_get_file_path(storage_path, cluster); + if xtream_path.exists() { + let _file_lock = app_config.file_locks.write_lock(&xtream_path).await; + let mut tree: BPlusTreeUpdate = BPlusTreeUpdate::try_new(&xtream_path) + .map_err(|err| Error::other(format!("failed to open BPlusTree for input {input_name}: {err}")))?; + + let mut updated_plis = Vec::with_capacity(updates.len()); + for (provider_id, props) in updates { + match tree.query(&provider_id) { + Ok(Some(mut pli)) => { + pli.additional_properties = Some(props); + updated_plis.push((provider_id, pli)); + } + Ok(None) => { + error!("Could not find input entry for provider_id: {provider_id} and input: {input_name}"); + } + Err(err) => { + error!("Failed to query BPlusTree for provider_id: {provider_id} and input: {input_name}: {err}"); + } + } + } + + if !updated_plis.is_empty() { + let refs: Vec<(&u32, &XtreamPlaylistItem)> = updated_plis.iter() + .map(|(id, pli)| (id, pli)) + .collect(); + tree.update_batch(&refs).map_err(|err| Error::other(format!("failed to write batch {cluster} info for input {input_name}: {err}")))?; + } + } + Ok(()) +} + pub async fn persist_input_vod_info(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, input_name: &str, provider_id: u32, @@ -809,12 +841,30 @@ pub async fn persist_input_vod_info(app_config: &Arc, storage_path: & persist_input_info(app_config, storage_path, cluster, input_name, provider_id, StreamProperties::Video(Box::new(props.clone()))).await } +pub async fn persist_input_vod_info_batch(app_config: &Arc, storage_path: &Path, + cluster: XtreamCluster, input_name: &str, + updates: Vec<(u32, VideoStreamProperties)>) -> Result<(), Error> { + let batch = updates.into_iter() + .map(|(id, props)| (id, StreamProperties::Video(Box::new(props)))) + .collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} + pub async fn persists_input_series_info(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, input_name: &str, provider_id: u32, props: &SeriesStreamProperties) -> Result<(), Error> { persist_input_info(app_config, storage_path, cluster, input_name, provider_id, StreamProperties::Series(Box::new(props.clone()))).await } +pub async fn persist_input_series_info_batch(app_config: &Arc, storage_path: &Path, + cluster: XtreamCluster, input_name: &str, + updates: Vec<(u32, SeriesStreamProperties)>) -> Result<(), Error> { + let batch = updates.into_iter() + .map(|(id, props)| (id, StreamProperties::Series(Box::new(props)))) + .collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} + pub async fn load_input_xtream_playlist(app_config: &Arc, storage_path: &Path, clusters: &[XtreamCluster]) -> Result, TuliproxError> { let mut groups: IndexMap = IndexMap::new(); @@ -851,7 +901,7 @@ pub async fn load_input_xtream_playlist(app_config: &Arc, storage_pat groups.entry(cat_id) .or_insert_with(|| PlaylistGroup { id: cat_id, - title: String::from("Unknown"), + title: intern("Unknown"), channels: Vec::new(), xtream_cluster: cluster, }) diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index afff6c676..9f73999d1 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -11,7 +11,7 @@ use arc_swap::{ArcSwap, ArcSwapAny}; use chrono::Local; use log::{error, info, warn}; use serde::Serialize; -use shared::error::{create_tuliprox_error, info_err, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err, info_err_res, TuliproxError}; use shared::model::{ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, SourcesConfigDto, TargetUserDto}; use shared::utils::CONSTANTS; use std::env; @@ -72,15 +72,15 @@ pub fn read_sources_file_from_path(sources_file: &Path, resolve_env: bool, inclu Ok(mut sources) => { if resolve_env { if let Err(err) = sources.prepare(include_computed, hdhr_config) { - return Err(info_err!(format!("Can't read the sources-config file: {}: {err}", sources_file.display()))); + return info_err_res!("Can't read the sources-config file: {}: {err}", sources_file.display()); } } Ok(sources) } - Err(err) => Err(info_err!(format!("Can't read the sources-config file: {}: {err}", sources_file.display()))) + Err(err) => info_err_res!("Can't read the sources-config file: {}: {err}", sources_file.display()) } } - Err(err) => Err(info_err!(format!("Can't read the sources-config file: {}: {err}", sources_file.display()))) + Err(err) => info_err_res!("Can't read the sources-config file: {}: {err}", sources_file.display()) } } @@ -100,10 +100,10 @@ pub fn read_config_file(config_file: &str, resolve_env: bool, include_computed: } Ok(config) } - Err(err) => Err(info_err!(format!("Can't read the config file: {config_file}: {err}"))) + Err(err) => info_err_res!("Can't read the config file: {config_file}: {err}") } } - Err(err) => Err(info_err!(format!("Can't read the config file: {config_file}: {err}"))) + Err(err) => info_err_res!("Can't read the config file: {config_file}: {err}") } } @@ -175,7 +175,7 @@ pub async fn get_batch_aliases(input_type: InputType, url: &str) -> Result { - Err(TuliproxError::new(TuliproxErrorKind::Info, err.to_string())) + info_err_res!("{err}") } }; } @@ -285,7 +285,7 @@ pub fn read_api_proxy_file(api_proxy_file: &str, resolve_env: bool) -> Result { - Err(info_err!(format!("cant read api-proxy-config file: {err}"))) + info_err_res!("cant read api-proxy-config file: {err}") } } }) @@ -319,8 +319,25 @@ where { let path = PathBuf::from(file_path); let filename = path.file_name().map_or(default_name.to_string(), |f| f.to_string_lossy().to_string()); - let backup_path = PathBuf::from(backup_dir).join(format!("{filename}_{}", Local::now().format("%Y%m%d_%H%M%S"))); + + let mut serialized = String::new(); + let options = serde_saphyr::SerializerOptions { + prefer_block_scalars: false, + ..Default::default() + }; + serde_saphyr::to_fmt_writer_with_options(&mut serialized, &config, options) + .map_err(|err| info_err!("Could not serialize config: {}", err))?; + if path.exists() { + if let Ok(existing) = fs::read_to_string(&path).await { + if existing == serialized { + // info!("File {} unchanged, skipping write", path.display()); + return Ok(()); + } + } + } + + let backup_path = PathBuf::from(backup_dir).join(format!("{filename}_{}", Local::now().format("%Y%m%d_%H%M%S"))); match fs::copy(&path, &backup_path).await { Ok(_) => {} @@ -328,17 +345,9 @@ where } info!("Saving file to {}", &path.to_str().unwrap_or("?")); - let mut serialized = String::new(); - let options = serde_saphyr::SerializerOptions { - prefer_block_scalars: false, - ..Default::default() - }; - serde_saphyr::to_fmt_writer_with_options(&mut serialized, &config, options) - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Info, "Could not serialize config: {}", err))?; - fs::write(&path, serialized) .await - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Info, "Could not write file {}: {}", &path.to_str().unwrap_or("?"), err)) + .map_err(|err| info_err!("Could not write file {}: {}", &path.to_str().unwrap_or("?"), err)) } pub async fn save_api_proxy(file_path: &str, backup_dir: &str, config: &ApiProxyConfigDto) -> Result<(), TuliproxError> { diff --git a/backend/src/utils/file/mapping_reader.rs b/backend/src/utils/file/mapping_reader.rs index d3eccbf54..49980c216 100644 --- a/backend/src/utils/file/mapping_reader.rs +++ b/backend/src/utils/file/mapping_reader.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; use crate::model::{Mappings}; -use shared::error::{create_tuliprox_error_result, info_err, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, TuliproxError}; use crate::utils::traverse_dir; use crate::utils::{config_file_reader, open_file}; use log::{warn}; use std::path::{Path, PathBuf}; +use shared::info_err; use shared::model::{MappingDefinitionDto, MappingDto, MappingsDto, PatternTemplate}; fn read_mapping(mapping_file: &Path, resolve_var: bool, prepare_mappings: bool) -> Result, TuliproxError> { @@ -18,7 +19,7 @@ fn read_mapping(mapping_file: &Path, resolve_var: bool, prepare_mappings: bool) Ok(Some(mapping)) } Err(err) => { - Err(info_err!(err.to_string())) + info_err_res!("{err}") } }; } @@ -91,9 +92,9 @@ fn read_mappings_from_directory(path: &Path, resolve_env: bool) -> Result