From 8cc9283a31b0c4d071d7807e422316f87b41d8b5 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 8 Jan 2026 15:11:12 +0100 Subject: [PATCH] Playlist Update Optimization: Reduced Memory Usage An optimization has been introduced to reduce memory consumption during playlist updates. Overview Previously, provider playlists were fully loaded into memory during the update process. For large playlists (e.g. hundreds of thousands of entries or multiple providers), this could result in significant RAM usage. With the new implementation, it is now possible to optionally read provider playlists directly from disk instead of keeping them entirely in memory. How It Works - users can configure whether: - Provider playlists are loaded into memory (previous behavior), or - Provider playlists are streamed to/from disk to minimize RAM usage. Processing remains sequential and batch-based, ensuring identical functional behavior. This approach significantly lowers peak memory usage, especially on systems with limited resources. Benefits - Reduced peak RAM consumption during playlist updates - Better scalability for large playlists and multiple providers - Full backward compatibility Trade-offs / Drawbacks - Increased processing time due to reduced in-memory caching - Higher disk I/O usage, especially for large or fragmented playlists - Performance depends more strongly on disk speed (Nvme SSD, HDD) - Slightly increased CPU overhead due to repeated parsing and deserialization - Not optimal for environments where fast updates are more important than memory usage Recommendation - Use in-memory mode for systems with sufficient RAM and a focus on update speed - Use disk-based mode for large playlists, multiple providers, or memory-constrained environments --- CHANGELOG.md | 3 + Cargo.lock | 44 +- README.md | 1 + backend/Cargo.toml | 12 +- backend/src/api/config_file.rs | 1 + backend/src/api/config_watch.rs | 52 +- .../src/api/endpoints/api_playlist_utils.rs | 11 +- backend/src/api/endpoints/hls_api.rs | 5 +- backend/src/api/endpoints/m3u_api.rs | 2 +- backend/src/api/endpoints/user_api.rs | 18 +- backend/src/api/endpoints/v1_api.rs | 2 +- backend/src/api/endpoints/v1_api_config.rs | 2 +- backend/src/api/endpoints/xtream_api.rs | 32 +- backend/src/api/model/active_user_manager.rs | 4 +- backend/src/api/model/model_utils.rs | 4 +- backend/src/api/model/xtream.rs | 31 +- backend/src/api/panel_api.rs | 48 +- backend/src/library/metadata_resolver.rs | 5 +- backend/src/library/scanner.rs | 5 +- backend/src/main.rs | 4 +- backend/src/model/config/app.rs | 16 +- backend/src/model/config/base.rs | 2 + backend/src/model/config/favourites.rs | 3 +- backend/src/model/config/input.rs | 4 +- backend/src/model/config/library.rs | 16 +- backend/src/model/config/panel_api.rs | 38 +- backend/src/model/config/source.rs | 6 +- backend/src/model/config/web_auth.rs | 6 +- backend/src/model/playlist.rs | 61 +- backend/src/processing/mod.rs | 5 +- backend/src/processing/parser/m3u.rs | 67 +- backend/src/processing/parser/xtream.rs | 220 +++++- backend/src/processing/processor/epg.rs | 34 +- backend/src/processing/processor/library.rs | 36 +- backend/src/processing/processor/playlist.rs | 456 ++++++----- backend/src/processing/processor/sort.rs | 6 +- backend/src/processing/processor/trakt.rs | 13 +- backend/src/processing/processor/xtream.rs | 4 +- .../src/processing/processor/xtream_series.rs | 172 +++-- .../src/processing/processor/xtream_vod.rs | 89 ++- backend/src/repository/bplustree.rs | 728 +++++++++++++----- backend/src/repository/epg_repository.rs | 16 +- backend/src/repository/library_repository.rs | 54 ++ .../src/repository/m3u_playlist_iterator.rs | 10 +- backend/src/repository/m3u_repository.rs | 15 +- backend/src/repository/mod.rs | 4 + backend/src/repository/playlist_repository.rs | 115 ++- backend/src/repository/playlist_source.rs | 525 +++++++++++++ backend/src/repository/storage.rs | 23 +- backend/src/repository/strm_repository.rs | 26 +- .../repository/xtream_playlist_iterator.rs | 11 +- backend/src/repository/xtream_repository.rs | 128 ++- backend/src/utils/file/config_reader.rs | 45 +- backend/src/utils/file/mapping_reader.rs | 11 +- backend/src/utils/geoip.rs | 40 +- backend/src/utils/network/epg.rs | 70 +- backend/src/utils/network/request.rs | 98 +-- backend/src/utils/network/xtream.rs | 229 +++++- backend/src/utils/trakt/client.rs | 6 +- backend/src/utils/trakt/errors.rs | 10 +- frontend/Cargo.toml | 8 +- frontend/public/assets/i18n/en.json | 44 +- frontend/public/assets/icons.json | 4 + frontend/scss/_theme.scss | 3 + .../scss/app/components/_icon_button.scss | 7 + .../app/components/config/_config_view.scss | 27 +- .../app/components/playlist/_epg_view.scss | 4 +- .../src/app/components/api_user/playlist.rs | 5 +- .../src/app/components/config/config_page.rs | 38 +- .../app/components/config/config_update.rs | 1 + .../src/app/components/config/config_view.rs | 12 +- .../components/config/library_config_view.rs | 432 +++++++++++ .../app/components/config/log_config_view.rs | 10 +- .../app/components/config/main_config_view.rs | 4 + frontend/src/app/components/config/mod.rs | 2 + .../components/config/panel_config_view.rs | 93 +-- .../components/config/webui_config_view.rs | 7 +- .../app/components/dashboard/streams_table.rs | 6 +- frontend/src/app/components/icon_button.rs | 2 +- .../src/app/components/playlist/epg_view.rs | 5 +- .../app/components/playlist/input_table.rs | 4 +- .../playlist/playlist_editor_page.rs | 5 +- .../components/playlist/playlist_explorer.rs | 5 +- .../playlist/playlist_explorer_page.rs | 5 +- .../app/components/playlist/target_table.rs | 4 +- frontend/src/app/components/theme.rs | 5 +- frontend/src/app/components/userlist/page.rs | 5 +- .../src/app/components/userlist/user_table.rs | 4 +- frontend/src/model/explorer_source_type.rs | 4 +- frontend/src/model/view_type.rs | 4 +- frontend/src/utils/mod.rs | 2 +- shared/Cargo.toml | 8 +- shared/src/error/tuliprox_error.rs | 44 +- shared/src/foundation/filter.rs | 70 +- shared/src/foundation/mapper.rs | 66 +- shared/src/model/config/api_proxy.rs | 12 +- shared/src/model/config/base.rs | 7 + shared/src/model/config/cache.rs | 6 +- shared/src/model/config/epg.rs | 4 +- shared/src/model/config/epg_smart_match.rs | 4 +- shared/src/model/config/favourites.rs | 5 +- shared/src/model/config/hdhomerun.rs | 12 +- shared/src/model/config/input.rs | 20 +- shared/src/model/config/library.rs | 98 ++- shared/src/model/config/macros.rs | 22 +- shared/src/model/config/proxy_type.rs | 5 +- shared/src/model/config/proxy_user_status.rs | 6 +- shared/src/model/config/rename.rs | 4 +- shared/src/model/config/sort.rs | 6 +- shared/src/model/config/source.rs | 10 +- shared/src/model/config/stream.rs | 12 +- shared/src/model/config/target.rs | 34 +- shared/src/model/config/video_download.rs | 6 +- shared/src/model/item_field.rs | 6 +- shared/src/model/mapping.rs | 25 +- shared/src/model/messaging.rs | 6 +- shared/src/model/playlist.rs | 95 ++- shared/src/model/playlist_request.rs | 14 +- shared/src/model/stream_info.rs | 3 +- shared/src/utils/default_utils.rs | 4 +- shared/src/utils/mod.rs | 2 + shared/src/utils/string_interner.rs | 81 ++ 122 files changed, 3765 insertions(+), 1397 deletions(-) create mode 100644 backend/src/repository/library_repository.rs create mode 100644 backend/src/repository/playlist_source.rs create mode 100644 frontend/src/app/components/config/library_config_view.rs create mode 100644 shared/src/utils/string_interner.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddad6739..c56387942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - **Panel API Integration**: Optional integration to renew expired accounts or provision new aliases when connections are exhausted. - **Playlist Caching**: Added `cache_duration` to inputs, allowing configurable provider playlist cache times during subsequent updates (e.g., `60s`, `5m` `12h`, `1d`). - **Database Viewer**: New CLI flags `--dbx` and `--dbm` to inspect internal database content. +- **Added `disk_based_processing`** (boolean, default `false`) to `config.yml`. When enabled, input playlists are processed from disk instead of memory. ## ⚙️ Engine & Storage Optimizations - **Slotted Page Architecture**: Improved space utilization and support for variable-length keys. @@ -30,6 +31,8 @@ - **Atomic I/O Layer**: Refactored for atomic writes and file locking, ensuring data integrity. - **B+Tree Compaction**: Reclaim space after deletions or mass updates. - **Batch Upsert**: Significantly higher throughput during mass inserts/updates. +- **Disk-Based Provider Processing**: New `disk_based_processing` config option massively reduces RAM usage by streaming playlist data from disk (BPlusTree) during updates. +- **String Interning**: Implemented `Arc` string interning for playlist items to further reduce memory footprint. ## 🔍 Mapping & Filtering Enhancements - **Accent-Independent Matching**: Integrated `match_as_ascii` flag for robust text matching (e.g., "Cinema" matches "Cinéma"). diff --git a/Cargo.lock b/Cargo.lock index dc33848ee..29572b2bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1156,7 +1156,7 @@ dependencies = [ [[package]] name = "frontend" -version = "3.2.37" +version = "3.2.38" dependencies = [ "anyhow", "base64", @@ -1168,9 +1168,9 @@ dependencies = [ "futures-signals", "gloo-render 0.2.0", "gloo-storage 0.3.0", - "gloo-timers 0.2.6", + "gloo-timers 0.3.0", "gloo-utils 0.2.0", - "implicit-clone", + "implicit-clone 0.6.0", "js-sys", "log", "prost", @@ -1691,8 +1691,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" dependencies = [ - "futures-channel", - "futures-core", "js-sys", "wasm-bindgen", ] @@ -2252,6 +2250,15 @@ dependencies = [ "indexmap", ] +[[package]] +name = "implicit-clone" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1689b939ee35e3a075b0834b5672efd43aec8a6e81a1c6002b76a5ca2f211ae0" +dependencies = [ + "implicit-clone-derive", +] + [[package]] name = "implicit-clone-derive" version = "0.1.2" @@ -2505,15 +2512,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lz4_flex" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" -dependencies = [ - "twox-hash", -] - [[package]] name = "lz4_flex" version = "0.12.0" @@ -3863,9 +3861,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "0.0.12" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef7e021087b4b525edcbb93045ca3f1dafeff7800b3c7960a90a75f197e257da" +checksum = "be7f28bb35aab6f9cdc5f464f9bbda0b1e3908766e819557111cc13e58ce7915" dependencies = [ "ahash", "annotate-snippets", @@ -3992,7 +3990,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.2.37" +version = "3.2.38" dependencies = [ "base64", "bitflags 2.10.0", @@ -4008,7 +4006,7 @@ dependencies = [ "indexmap", "js-sys", "log", - "lz4_flex 0.12.0", + "lz4_flex", "path-clean", "pest", "pest_derive", @@ -4566,7 +4564,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.2.37" +version = "3.2.38" dependencies = [ "arc-swap", "async-compression", @@ -4597,7 +4595,7 @@ dependencies = [ "libc", "log", "lol_html", - "lz4_flex 0.11.5", + "lz4_flex", "mime", "mime_guess", "notify", @@ -4697,9 +4695,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -5351,7 +5349,7 @@ dependencies = [ "console_error_panic_hook", "futures", "gloo 0.10.0", - "implicit-clone", + "implicit-clone 0.4.9", "indexmap", "js-sys", "prokio", diff --git a/README.md b/README.md index baed2387d..d0694072f 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Top level entries in the config files are: * `config_hot_reload` _optional_, default false. * `sleep_timer_mins` _optional_, used for closing stream after the given minutes. * `accept_unsecure_ssl_certificates` _optional_, default false. +* `disk_based_processing` _optional_, default false. When set to true, input playlists are processed from disk to save RAM. * `library` _optional_, for local media ### 1.1. `process_parallel` diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2d1545c3b..94757ca35 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "tuliprox" -version = "3.2.37" +version = "3.2.38" edition = "2021" -rust-version = "1.87.0" +rust-version = "1.89.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,13 +11,13 @@ lol_html = "2.7" handlebars = "6" shared = { version = "3", path = "../shared" } serde = { version = "1.0", features = ["derive", "rc"] } -serde-saphyr = "0.0.12" +serde-saphyr = "0.0.13" serde_json = { version = "1", features = ["raw_value"] } quick-xml = { version = "0.38", features = ["async-tokio"] } regex = "1.12" clap = { version = "4", features = ["derive"] } reqwest = { version = "0", features = ["blocking", "json", "stream", "rustls-tls", "socks"] } -url = "2.5.7" +url = "2.5.8" chrono = "0.4" cron = "0.15" axum = { version = "0.8" , features = ["macros", "default", "ws"]} @@ -44,7 +44,7 @@ flate2 = "1.1" blake3 = "1.8" bytes = "1.11" tokio-stream = { version = "0.1", features = ["sync"] } -tokio = { version = "1.48", features = ["rt-multi-thread", "parking_lot", "fs"] } +tokio = { version = "1.49", features = ["rt-multi-thread", "parking_lot", "fs"] } #tokio = { version = "1.46", features = ["rt-multi-thread", "parking_lot", "fs", "tracing"] } #console-subscriber = "0" #tracing = "0.1" @@ -58,7 +58,7 @@ rayon = "1.11" base64 = "0.22" paste = "1.0" indexmap = "2" -lz4_flex = "0.11" +lz4_flex = "0.12" fs2 = "0.4" notify = "8.2" arc-swap = "1.8" diff --git a/backend/src/api/config_file.rs b/backend/src/api/config_file.rs index 6c25730eb..533d02661 100644 --- a/backend/src/api/config_file.rs +++ b/backend/src/api/config_file.rs @@ -10,6 +10,7 @@ use shared::model::{ConfigPaths, ConfigType}; use std::path::Path; use std::sync::Arc; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ConfigFile { Config, ApiProxy, diff --git a/backend/src/api/config_watch.rs b/backend/src/api/config_watch.rs index eb829c764..2de1419c0 100644 --- a/backend/src/api/config_watch.rs +++ b/backend/src/api/config_watch.rs @@ -39,10 +39,6 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke } }); - // Use recommended_watcher() to automatically select the best implementation - // for your platform. The `EventHandler` passed to this constructor can be a - // closure, a `std::sync::mpsc::Sender`, a `crossbeam_channel::Sender`, or - // another type the trait is implemented for. let mut watcher = recommended_watcher(std_tx).map_err(|err| TuliproxError::new(TuliproxErrorKind::Info, format!("Failed to init config file watcher {err}")))?; watcher.watch(path, recursive_mode).map_err(|err| TuliproxError::new(TuliproxErrorKind::Info, format!("Failed to start config file watcher {err}")))?; info!("Watching config file changes {}", path.display()); @@ -62,6 +58,10 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke let _keep_watcher_alive = watcher; + let mut debounce_timer = Box::pin(tokio::time::sleep(tokio::time::Duration::from_millis(0))); + let mut timer_active = false; + let mut pending_configs: HashMap = HashMap::new(); + loop { tokio::select! { biased; @@ -75,27 +75,49 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke match res { Ok(event) => { if let EventKind::Access(AccessKind::Close(AccessMode::Write)) = event.kind { - for path in event.paths { - if let Some((config_file, _is_dir)) = files.get(&path) { - if let Err(err) = config_file.reload(&path, &watcher_app_state).await { - handle_error(err, &path); - } - } else if recursive_mode == RecursiveMode::Recursive && path.extension().is_some_and(|ext| ext == "yml") { - for (key, (config_file, is_dir)) in &files { - if *is_dir && path.starts_with(key) { - if let Err(err) = config_file.reload(&path, &watcher_app_state).await { - handle_error(err, &path); + let mut trigger = false; + for path in event.paths { + let mut resolved = None; + if let Some((config_file, _is_dir)) = files.get(&path) { + resolved = Some(*config_file); + } else if recursive_mode == RecursiveMode::Recursive && path.extension().is_some_and(|ext| ext == "yml") { + for (key, (config_file, is_dir)) in &files { + if *is_dir && path.starts_with(key) { + resolved = Some(*config_file); + break; } } } + + if let Some(config_file) = resolved { + // Consolidate updates + let effective = if config_file == ConfigFile::SourceFile { ConfigFile::Sources } else { config_file }; + pending_configs.insert(effective, path); + trigger = true; + } + } + + if trigger { + debounce_timer = Box::pin(tokio::time::sleep(tokio::time::Duration::from_millis(500))); + timer_active = true; } } } - } Err(e) => error!("watch error: {e:?}"), } } + () = &mut debounce_timer, if timer_active => { + timer_active = false; + if !pending_configs.is_empty() { + for (config_file, path) in pending_configs.drain() { + if let Err(err) = config_file.reload(&path, &watcher_app_state).await { + handle_error(err, &path); + } + } + } + } + else => break, // Channel closed } } diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index e4f8e861d..d594411d1 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -14,12 +14,12 @@ use crate::api::api_utils::{json_or_bin_response}; fn group_playlist_items( cluster: XtreamCluster, iter: impl Iterator, - get_group: fn(&T) -> String, + get_group: fn(&T) -> Arc, ) -> Vec where T: Serialize + Into, { - let mut groups: IndexMap> = IndexMap::new(); + let mut groups: IndexMap, Vec> = IndexMap::new(); for item in iter { let group_key = get_group(&item); @@ -115,7 +115,7 @@ async fn grouped_channels( )) } -pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&str>) -> impl axum::response::IntoResponse + Send { +pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&str>) -> impl IntoResponse + Send { if let Some(target) = cfg_target { if target.has_output(TargetType::Xtream) { let live_channels = grouped_channels(cfg, target, XtreamCluster::Live).await; @@ -150,7 +150,10 @@ pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cf let (result, errors) = match input.input_type { InputType::M3u | InputType::M3uBatch => m3u::download_m3u_playlist(client, cfg, input).await, - InputType::Xtream | InputType::XtreamBatch => xtream::download_xtream_playlist(cfg, client, input, None).await, + InputType::Xtream | InputType::XtreamBatch => { + let (pl, err, _) = xtream::download_xtream_playlist(cfg, client, input, None).await; + (pl, err) + } InputType::Library => { return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({ "error": "Library inputs are not supported on this endpoint"}))).into_response(); } diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 05a0e0030..7e5de296d 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -21,7 +21,7 @@ use axum::response::IntoResponse; use log::{debug, error}; use serde::Deserialize; use shared::model::{PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, XtreamCluster}; -use shared::utils::{is_hls_url, replace_url_extension, sanitize_sensitive_info, CUSTOM_VIDEO_PREFIX, HLS_EXT}; +use shared::utils::{intern, is_hls_url, replace_url_extension, sanitize_sensitive_info, CUSTOM_VIDEO_PREFIX, HLS_EXT}; use std::sync::Arc; const PLAYLIST_TEMPLATE: &str = r"#EXTM3U @@ -132,6 +132,7 @@ pub(in crate::api) async fn handle_hls_stream_request( &input_source, Some(&headers), None, + false ) .await { @@ -196,7 +197,7 @@ async fn resolve_stream_channel( provider_id: 0, item_type: PlaylistItemType::LiveHls, cluster: XtreamCluster::Live, - group: "Unknown".to_string(), + group: intern("Unknown"), title: "Unknown".to_string(), url: hls_url.to_string(), shared: false, diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index 3526a83f7..d8d8545bd 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -126,7 +126,7 @@ async fn m3u_api_stream( let input = try_option_bad_request!( app_state .app_config - .get_input_by_name(pli.input_name.as_str()), + .get_input_by_name(&pli.input_name), true, format!("Cant find input {} for target {target_name}, stream_id {virtual_id}", pli.input_name) ); diff --git a/backend/src/api/endpoints/user_api.rs b/backend/src/api/endpoints/user_api.rs index 03a8646bc..48fab2dc3 100644 --- a/backend/src/api/endpoints/user_api.rs +++ b/backend/src/api/endpoints/user_api.rs @@ -1,20 +1,20 @@ +use crate::api::api_utils::try_unwrap_body; use crate::api::api_utils::{get_user_target_by_username, get_username_from_auth_header}; use crate::api::model::AppState; use crate::auth::validator_user; -use crate::model::{AppConfig, ConfigTarget}; -use shared::model::{PlaylistBouquetDto, TargetType, XtreamCluster}; +use crate::auth::AuthBearer; use crate::model::PlaylistXtreamCategory; +use crate::model::{AppConfig, ConfigTarget}; +use crate::repository::m3u_repository; use crate::repository::user_repository::{load_user_bouquet_as_json, save_user_bouquet}; use crate::repository::xtream_repository::xtream_get_playlist_categories; -use crate::repository::m3u_repository; +use axum::response::IntoResponse; use bytes::Bytes; use futures::{stream, StreamExt}; use log::error; +use shared::model::{PlaylistBouquetDto, TargetType, XtreamCluster}; use std::collections::HashSet; use std::sync::Arc; -use axum::response::IntoResponse; -use crate::auth::AuthBearer; -use crate::api::api_utils::try_unwrap_body; fn get_categories_from_xtream(categories: Option>) -> Vec { let mut groups: Vec = Vec::new(); @@ -27,12 +27,12 @@ fn get_categories_from_xtream(categories: Option>) - } -async fn get_categories_from_m3u_playlist(target: &ConfigTarget, config: &AppConfig) -> Vec { +async fn get_categories_from_m3u_playlist(target: &ConfigTarget, config: &AppConfig) -> Vec> { let mut groups = Vec::new(); if let Some((_guard, iter)) = m3u_repository::iter_raw_m3u_playlist(config, target).await { let mut unique_groups = HashSet::new(); for (item, _has_next) in iter { - if !unique_groups.contains(item.group.as_str()) { + if !unique_groups.contains(&item.group) { unique_groups.insert(item.group.clone()); groups.push(item.group.clone()); } @@ -153,7 +153,7 @@ pub fn user_api_register(app_state: Arc) -> axum::Router .route("/playlist/categories", axum::routing::get(playlist_categories)) .route("/playlist/bouquet", axum::routing::get(playlist_bouquet)) .route("/playlist/bouquet", axum::routing::post(save_playlist_bouquet)) - .route_layer(axum::middleware::from_fn_with_state(app_state, validator_user)) + .route_layer(axum::middleware::from_fn_with_state(app_state, validator_user)), ) diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index 1e010bab7..e06f103aa 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -91,7 +91,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State { let reader = Cursor::new(content); let mut geoip = GeoIp::new(); diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 2c82855ea..9622d0e2c 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -148,7 +148,7 @@ async fn config_batch_content( if let Some(batch_url) = config_input.t_batch_url.as_ref() { let input_source = InputSource::from(&*config_input).with_url(batch_url.to_owned()); let disabled_headers = app_state.get_disabled_headers(); - return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await { + return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None, false).await { Ok((content, _path)) => { // Return CSV with explicit content-type try_unwrap_body!(axum::response::Response::builder() diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index 1d5c9f864..a4be32d06 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1,4 +1,5 @@ // https://github.com/tellytv/go.xtream-codes/blob/master/structs.go +// Xtream api -> https://9tzx6f0ozj.apidog.io/ use crate::api::api_utils; use crate::api::api_utils::{create_session_fingerprint, local_stream_response, try_unwrap_body}; use crate::api::api_utils::{ @@ -32,9 +33,7 @@ use futures::Stream; use log::{debug, error, warn}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use shared::error::create_tuliprox_error_result; -use shared::error::info_err; -use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, info_err, TuliproxError}; use shared::model::{create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType, TargetType, UserConnectionPermission, XtreamCluster}; use shared::utils::{ deserialize_as_string, extract_extension_from_url, generate_playlist_uuid, @@ -92,11 +91,7 @@ impl FromStr for ApiStreamContext { Self::MOVIE => Ok(Self::Movie), Self::SERIES => Ok(Self::Series), Self::TIMESHIFT => Ok(Self::Timeshift), - _ => create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "Unknown ApiStreamContext: {}", - s - ), + _ => info_err_res!("Unknown ApiStreamContext: {}", s), } } } @@ -255,7 +250,7 @@ async fn xtream_player_api_stream( } let input = try_option_bad_request!( - app_state.app_config.get_input_by_name(pli.input_name.as_str()), + app_state.app_config.get_input_by_name(&pli.input_name), true, format!( "Cant find input {} for target {target_name}, context {}, stream_id {virtual_id}", pli.input_name, stream_req.context) ); @@ -447,7 +442,7 @@ async fn xtream_player_api_stream_with_token( let input = try_option_bad_request!( app_state .app_config - .get_input_by_name(pli.input_name.as_str()), + .get_input_by_name(&pli.input_name), true, format!( "Cant find input {} for target {target_name}, context {}, stream_id {}", @@ -829,7 +824,7 @@ async fn xtream_get_stream_info_response( Some(cluster), ).await { if pli.item_type.is_local() { - let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name))) else { + let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected: xtream output required for target {}", target.name)) else { return try_unwrap_body!(empty_json_response()); }; @@ -840,7 +835,7 @@ async fn xtream_get_stream_info_response( if pli.provider_id > 0 { let input_name = &pli.input_name; - if let Some(input) = app_state.app_config.get_input_by_name(input_name.as_str()) { + if let Some(input) = app_state.app_config.get_input_by_name(input_name) { if let Some(info_url) = xtream::get_xtream_player_api_info_url(&input, cluster, pli.provider_id) { // Redirect is only possible for live streams, vod and series info needs to be modified if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live { @@ -925,7 +920,7 @@ async fn xtream_get_short_epg( if pli.provider_id > 0 { let input_name = &pli.input_name; - if let Some(input) = app_state.app_config.get_input_by_name(input_name.as_str()) { + if let Some(input) = app_state.app_config.get_input_by_name(input_name) { if let Some(action_url) = xtream::get_xtream_player_api_action_url( &input, crate::model::XC_ACTION_GET_SHORT_EPG, @@ -952,6 +947,7 @@ async fn xtream_get_short_epg( &input_source, None, None, + false ) .await { @@ -1039,7 +1035,7 @@ async fn xtream_get_catchup_response( ).await); let input = try_option_bad_request!(app_state .app_config - .get_input_by_name(pli.input_name.as_str())); + .get_input_by_name(&pli.input_name)); let info_url = try_option_bad_request!(xtream::get_xtream_player_api_action_url( &input, crate::model::XC_ACTION_GET_CATCHUP_TABLE @@ -1053,7 +1049,8 @@ async fn xtream_get_catchup_response( let content = try_result_bad_request!( xtream::get_xtream_stream_info_content( &app_state.http_client.load(), - &input_source + &input_source, + false ) .await ); @@ -1288,10 +1285,7 @@ async fn xtream_player_api( ) .await ), - _ => Some(Err(info_err!(format!( - "Cant find content: {action} for target: {}", - &target.name - )))), + _ => Some(info_err_res!("Cant find content: {action} for target: {}", &target.name)), }; match result { diff --git a/backend/src/api/model/active_user_manager.rs b/backend/src/api/model/active_user_manager.rs index bcb71966a..3088f1360 100644 --- a/backend/src/api/model/active_user_manager.rs +++ b/backend/src/api/model/active_user_manager.rs @@ -7,7 +7,7 @@ use arc_swap::ArcSwapOption; use jsonwebtoken::get_current_timestamp; use log::{debug, info}; use shared::model::{ActiveUserConnectionChange, StreamChannel, StreamInfo, UserConnectionPermission, VirtualId}; -use shared::utils::{current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, sanitize_sensitive_info, strip_port}; +use shared::utils::{current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, intern, sanitize_sensitive_info, strip_port}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; @@ -251,7 +251,7 @@ impl ActiveUserManager { if &stream.addr == addr { stream.provider = String::from("tuliprox"); stream.channel.title = video_type.to_string(); - stream.channel.group = String::new(); + stream.channel.group = intern(""); return Some(stream.clone()); } } diff --git a/backend/src/api/model/model_utils.rs b/backend/src/api/model/model_utils.rs index 718d4be61..f20c2ea29 100644 --- a/backend/src/api/model/model_utils.rs +++ b/backend/src/api/model/model_utils.rs @@ -5,13 +5,12 @@ use reqwest::header::HeaderMap; use shared::utils::{filter_response_header}; pub fn get_response_headers(headers: &HeaderMap) -> Vec<(String, String)> { - let mut response_headers: Vec<(String, String)> = headers.iter() + let response_headers: Vec<(String, String)> = headers.iter() .filter(|(key, _)| filter_response_header(key.as_str())) .filter_map(|(key, value)| { value.to_str().ok().map(|v| (key.to_string(), v.to_string())) }) .collect(); - response_headers.push((axum::http::header::CONNECTION.as_str().to_string(), "close".to_string())); response_headers } @@ -32,7 +31,6 @@ pub fn get_stream_response_with_headers(custom: Option<(Vec<(String, String)>, S let default_headers = vec![ ("content-type", "application/octet-stream"), - ("connection", "close"), ]; for (key, value) in default_headers { diff --git a/backend/src/api/model/xtream.rs b/backend/src/api/model/xtream.rs index e26835283..fbf940dbb 100644 --- a/backend/src/api/model/xtream.rs +++ b/backend/src/api/model/xtream.rs @@ -7,19 +7,19 @@ use shared::utils::CONSTANTS; #[derive(Serialize, Deserialize, Clone)] pub struct XtreamUserInfoResponse { - pub password: String, pub username: String, - pub active_cons: String, - pub allowed_output_formats: Vec, + pub password: String, + pub message: String, pub auth: u16, // 0 | 1 - #[serde(serialize_with ="serialize_number_as_string")] - pub created_at: i64, //1623429679, + pub status: String, // "Active" #[serde(serialize_with ="serialize_number_as_string")] pub exp_date: i64, //1628755200, pub is_trial: String, // 0 | 1 + pub active_cons: String, + #[serde(serialize_with ="serialize_number_as_string")] + pub created_at: i64, //1623429679, pub max_connections: String, - pub message: String, - pub status: String, // "Active" + pub allowed_output_formats: Vec, } #[derive(Serialize, Deserialize, Clone)] @@ -32,6 +32,7 @@ pub struct XtreamServerInfoResponse { pub timezone: String, pub timestamp_now: i64, pub time_now: String, //"2021-06-28 17:07:37" + pub process: bool, } #[derive(Serialize, Deserialize, Clone)] @@ -72,17 +73,17 @@ impl XtreamAuthorizationResponse { Self { user_info: XtreamUserInfoResponse { - active_cons: format!("{active_connections}"), - allowed_output_formats: CONSTANTS.allowed_output_formats.clone(), + username: user.username.clone(), + password: user.password.clone(), + message: server_info.message.clone(), auth: 1, - created_at, + status: user_status.to_string(), exp_date, is_trial, + active_cons: active_connections.to_string(), + created_at, max_connections, - message: server_info.message.clone(), - password: user.password.clone(), - username: user.username.clone(), - status: user_status.to_string(), + allowed_output_formats: CONSTANTS.allowed_output_formats.clone(), }, server_info: XtreamServerInfoResponse { url: server_info.host.clone(), @@ -93,6 +94,8 @@ impl XtreamAuthorizationResponse { timezone: server_info.timezone.clone(), timestamp_now: now.timestamp(), time_now: now.format("%Y-%m-%d %H:%M:%S").to_string(), + // We don't know what this field is good for, but it is in the response from XtreamCodes, so we will include it. + process: true }, } } diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index f4bb32fe8..1e948ba60 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -5,7 +5,7 @@ use crate::utils::{debug_if_enabled, persist_source_config, read_sources_file_fr use crate::utils::get_csv_file_path; use log::{debug, error, warn}; use serde_json::Value; -use shared::error::{create_tuliprox_error_result, info_err, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, info_err, TuliproxError}; use shared::model::{ConfigInputAliasDto, InputType}; use shared::utils::{get_credentials_from_url, parse_timestamp, sanitize_sensitive_info, trim_last_slash}; use std::path::{Path, PathBuf}; @@ -88,17 +88,17 @@ fn resolve_query_params( if value.eq_ignore_ascii_case("auto") { if key.eq_ignore_ascii_case("api_key") { let Some(k) = api_key.filter(|s| !s.trim().is_empty()) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but panel_api.api_key is missing"); + return info_err_res!("panel_api: query param {key} uses 'auto' but panel_api.api_key is missing"); }; value = k.to_string(); } else if key.eq_ignore_ascii_case("username") { let Some((u, _)) = creds else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but no account username is available"); + return info_err_res!("panel_api: query param {key} uses 'auto' but no account username is available"); }; value = u.to_string(); } else if key.eq_ignore_ascii_case("password") { let Some((_, pw)) = creds else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: query param {key} uses 'auto' but no account password is available"); + return info_err_res!("panel_api: query param {key} uses 'auto' but no account password is available"); }; value = pw.to_string(); } @@ -109,7 +109,7 @@ fn resolve_query_params( } fn build_panel_url(base_url: &str, query_params: &[(String, String)]) -> Result { - let mut url = Url::parse(base_url).map_err(|e| info_err!(format!("panel_api: invalid url {base_url}: {e}")))?; + let mut url = Url::parse(base_url).map_err(|e| info_err!("panel_api: invalid url {base_url}: {e}"))?; { let mut pairs = url.query_pairs_mut(); for (k, v) in query_params { @@ -156,14 +156,14 @@ async fn panel_get_json(app_state: &AppState, url: Url) -> Result Result< let url = build_panel_url(cfg.url.as_str(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new response is not a JSON object/array"); + return info_err_res!("panel_api: client_new response is not a JSON object/array"); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new status=false"); + return info_err_res!("panel_api: client_new status=false"); } if let Some((u, p)) = extract_username_password_from_json(obj) { return Ok((u, p, None)); @@ -191,7 +191,7 @@ async fn panel_client_new(app_state: &AppState, cfg: &PanelApiConfig) -> Result< return Ok((u, p, base)); } } - create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new response missing username/password (and no parsable url)") + info_err_res!("panel_api: client_new response missing username/password (and no parsable url)") } async fn panel_client_renew(app_state: &AppState, cfg: &PanelApiConfig, username: &str, password: &str) -> Result<(), TuliproxError> { @@ -203,11 +203,11 @@ async fn panel_client_renew(app_state: &AppState, cfg: &PanelApiConfig, username let url = build_panel_url(cfg.url.as_str(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_renew response is not a JSON object/array"); + return info_err_res!("panel_api: client_renew response is not a JSON object/array"); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_renew status=false"); + return info_err_res!("panel_api: client_renew status=false"); } Ok(()) } @@ -221,11 +221,11 @@ async fn panel_client_info(app_state: &AppState, cfg: &PanelApiConfig, username: let url = build_panel_url(cfg.url.as_str(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_info response is not a JSON object/array"); + return info_err_res!("panel_api: client_info response is not a JSON object/array"); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_info status=false"); + return info_err_res!("panel_api: client_info status=false"); } let expire = obj.get("expire").and_then(|v| v.as_str()).unwrap_or_default().trim(); let parsed = parse_timestamp(expire).ok().flatten(); @@ -302,11 +302,11 @@ async fn patch_source_yml_add_alias( ) -> Result<(), TuliproxError> { let mut sources = match read_sources_file_from_path(source_file_path, false, false, None) { Ok(sources) => sources, - Err(e) => return Err(TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read source file: {e}"))), + Err(e) => return info_err_res!("panel_api: failed to read source file: {e}"), }; let Some(input) = sources.inputs.iter_mut().find(|i| i.name == input_name) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find input '{input_name}' in source.yml"); + return info_err_res!("panel_api: could not find input '{input_name}' in source.yml"); }; let alias = ConfigInputAliasDto { @@ -335,15 +335,15 @@ async fn patch_source_yml_update_exp_date( ) -> Result<(), TuliproxError> { let mut sources = match read_sources_file_from_path(source_file_path, false, false, None) { Ok(sources) => sources, - Err(e) => return Err(TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read source file: {e}"))), + Err(e) => return info_err_res!("panel_api: failed to read source file: {e}"), }; let Some(input) = sources.inputs.iter_mut().find(|i| i.name == input_name) else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find input '{input_name}' in source.yml"); + return info_err_res!("panel_api: could not find input '{input_name}' in source.yml"); }; if input.update_account_expiration_date(input_name, account_name, exp_date).is_err() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: could not find account '{account_name}' under input '{input_name}' in source.yml"); + return info_err_res!("panel_api: could not find account '{account_name}' under input '{input_name}' in source.yml"); } persist_source_config(app_state, Some(source_file_path), sources).await?; @@ -399,7 +399,7 @@ async fn patch_batch_csv_append( lines.push(record.join(";")); tokio::fs::write(csv_path, lines.join("\n") + "\n") .await - .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to write csv: {e}")))?; + .map_err(|e| info_err!("panel_api: failed to write csv: {e}"))?; Ok(()) } @@ -412,11 +412,11 @@ async fn patch_batch_csv_update_exp_date( ) -> Result<(), TuliproxError> { let raw = tokio::fs::read_to_string(csv_path) .await - .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to read csv: {e}")))?; + .map_err(|e| info_err!("panel_api: failed to read csv: {e}"))?; let mut lines: Vec = raw.lines().map(ToString::to_string).collect(); let header_line_idx = lines.iter().position(|l| l.trim_start().starts_with('#')); let Some(header_idx) = header_line_idx else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: csv missing header line"); + return info_err_res!("panel_api: csv missing header line"); }; let header = lines[header_idx].trim_start_matches('#').trim(); let cols: Vec = header.split(';').map(|s| s.trim().to_lowercase()).collect(); @@ -461,7 +461,7 @@ async fn patch_batch_csv_update_exp_date( lines[i] = fields.join(";"); tokio::fs::write(csv_path, lines.join("\n") + "\n") .await - .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("panel_api: failed to write csv: {e}")))?; + .map_err(|e| info_err!("panel_api: failed to write csv: {e}"))?; return Ok(()); } } diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index aa3ed211d..e1c4038c4 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -128,6 +128,7 @@ mod tests { use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; use std::path::PathBuf; use std::time::Duration; + use shared::utils::intern; use crate::library::{MediaClassification, MediaClassifier}; fn create_test_config(tmdb_enabled: bool) -> LibraryConfig { @@ -157,8 +158,8 @@ mod tests { formats: vec![], }, playlist: LibraryPlaylistConfig { - movie_category: "Movies".to_string(), - series_category: "Series".to_string(), + movie_category: intern("Movies"), + series_category: intern("Series"), }, } } diff --git a/backend/src/library/scanner.rs b/backend/src/library/scanner.rs index cd4ebb238..5bc88a62a 100644 --- a/backend/src/library/scanner.rs +++ b/backend/src/library/scanner.rs @@ -282,6 +282,7 @@ impl LibraryScanner { #[cfg(test)] mod tests { use super::*; + use shared::utils::intern; use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; fn create_test_config() -> LibraryConfig { @@ -311,8 +312,8 @@ mod tests { formats: vec![], }, playlist: LibraryPlaylistConfig { - movie_category: "Local Movies".to_string(), - series_category: "Local Series".to_string(), + movie_category: intern("Local Movies"), + series_category: intern("Local Series"), }, } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 87182603f..c54f23f56 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -150,13 +150,13 @@ async fn main() { } let app_config = utils::read_initial_app_config(&mut config_paths, true, true, args.server) .await - .unwrap_or_else(|err| exit!("{}", err)); + .unwrap_or_else(|err| exit!("{err}")); print_info(&app_config); let sources = > as Access>::load(&app_config.sources); let targets = sources .validate_targets(args.target.as_ref()) - .unwrap_or_else(|err| exit!("{}", err)); + .unwrap_or_else(|err| exit!("{err}")); if args.server { start_in_server_mode(Arc::new(app_config), Arc::new(targets)).await; diff --git a/backend/src/model/config/app.rs b/backend/src/model/config/app.rs index 207eca44a..0fd4def99 100644 --- a/backend/src/model/config/app.rs +++ b/backend/src/model/config/app.rs @@ -5,7 +5,7 @@ use arc_swap::access::Access; use arc_swap::{ArcSwap, ArcSwapOption}; use log::{error, warn}; use rand::Rng; -use shared::create_tuliprox_error_result; +use shared::info_err_res; use shared::error::{TuliproxError, TuliproxErrorKind}; use shared::model::ConfigPaths; use std::borrow::Cow; @@ -81,10 +81,10 @@ impl AppConfig { if let Some(username) = output_username { if let Some((_, config_target)) = self.get_target_for_username(username) { if config_target.name != target_name { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "User:{username} does not belong to target: {}", target_name); + return info_err_res!("User:{username} does not belong to target: {}", target_name); } } else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "User: {username} does not exist"); + return info_err_res!("User: {username} does not exist"); } Ok(()) } else { @@ -249,20 +249,20 @@ impl AppConfig { for input in &sources.inputs { let input_name = input.name.trim(); if input_name.is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input name required"); + return info_err_res!("input name required"); } if seen_names.contains(input_name) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input names should be unique: {}", input_name); + return info_err_res!("input names should be unique: {}", input_name); } seen_names.insert(input_name.to_string()); if let Some(aliases) = &input.aliases { for alias in aliases { let input_name = alias.name.trim().to_string(); if input_name.is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input name required"); + return info_err_res!("input name required"); } if seen_names.contains(&input_name) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "input and alias names should be unique: {}", input_name); + return info_err_res!("input and alias names should be unique: {}", input_name); } seen_names.insert(input_name.clone()); } @@ -280,7 +280,7 @@ impl AppConfig { if let Some(targets) = &schedule.targets { for target_name in targets { if !target_names.contains(target_name.as_str()) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Unknown target name in scheduler: {}", target_name); + return info_err_res!("Unknown target name in scheduler: {}", target_name); } } } diff --git a/backend/src/model/config/base.rs b/backend/src/model/config/base.rs index 91cf2d17d..3712377fd 100644 --- a/backend/src/model/config/base.rs +++ b/backend/src/model/config/base.rs @@ -58,6 +58,7 @@ pub struct Config { pub sleep_timer_mins: Option, pub update_on_boot: bool, pub config_hot_reload: bool, + pub disk_based_processing: bool, pub accept_insecure_ssl_certificates: bool, pub web_ui: Option, pub messaging: Option, @@ -134,6 +135,7 @@ impl From<&ConfigDto> for Config { fn from(dto: &ConfigDto) -> Self { Config { process_parallel: dto.process_parallel, + disk_based_processing: dto.disk_based_processing, api: ConfigApi::from(&dto.api), working_dir: dto.working_dir.clone(), backup_dir: dto.backup_dir.clone(), diff --git a/backend/src/model/config/favourites.rs b/backend/src/model/config/favourites.rs index aa47343d0..09000ab45 100644 --- a/backend/src/model/config/favourites.rs +++ b/backend/src/model/config/favourites.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use crate::model::macros; use regex::Regex; use shared::foundation::filter::{CompiledRegex, Filter}; @@ -5,7 +6,7 @@ use shared::model::{ConfigFavouritesDto, ItemField}; #[derive(Debug, Clone)] pub struct ConfigFavourites { - pub group: String, + pub group: Arc, pub filter: Filter, pub match_as_ascii: bool, } diff --git a/backend/src/model/config/input.rs b/backend/src/model/config/input.rs index 229c9c466..daeee59b0 100644 --- a/backend/src/model/config/input.rs +++ b/backend/src/model/config/input.rs @@ -5,7 +5,7 @@ use log::warn; use shared::error::TuliproxError; use shared::model::{ConfigInputAliasDto, ConfigInputDto, ConfigInputOptionsDto, InputFetchMethod, InputType, StagedInputDto}; use shared::utils::{get_credentials_from_url}; -use shared::{check_input_connections, info_err, write_if_some}; +use shared::{check_input_connections, info_err_res, write_if_some}; use shared::check_input_credentials; use std::collections::HashMap; use std::fmt; @@ -156,7 +156,7 @@ impl ConfigInput { if let Some(staged_input) = &mut self.staged { check_input_credentials!(staged_input, staged_input.input_type, false, true); if !matches!(staged_input.input_type, InputType::M3u | InputType::Xtream) { - return Err(info_err!("Staged input can only be from type m3u or xtream".to_owned())); + return info_err_res!("Staged input can only be from type m3u or xtream"); } } diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index 1ef46c5c0..1c6b5ce6e 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -1,9 +1,9 @@ use crate::model::macros; -use shared::error::TuliproxError; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; use std::path::PathBuf; -use shared::info_err; -use shared::utils::default_metadata_path; +use std::sync::Arc; +use shared::utils::{default_metadata_path, intern}; #[derive(Debug, Clone, Default)] pub struct LibraryScanDirectory { @@ -40,8 +40,8 @@ pub struct LibraryTmdbConfig { #[derive(Debug, Clone)] pub struct LibraryPlaylistConfig { - pub movie_category: String, - pub series_category: String, + pub movie_category: Arc, + pub series_category: Arc, } @@ -67,7 +67,7 @@ impl LibraryConfig { dir.path = path.to_string(); } } - Err(err) => return Err(info_err!(format!("Failed to canonicalize directory path: {err}"))), + Err(err) => return info_err_res!("Failed to canonicalize directory path: {err}"), } } } @@ -121,8 +121,8 @@ impl From<&LibraryConfigDto> for LibraryConfig { formats: dto.metadata.formats.clone(), }, playlist: LibraryPlaylistConfig { - movie_category: dto.playlist.movie_category.clone(), - series_category: dto.playlist.series_category.clone(), + movie_category: intern(&dto.playlist.movie_category), + series_category: intern(&dto.playlist.series_category), }, } } diff --git a/backend/src/model/config/panel_api.rs b/backend/src/model/config/panel_api.rs index feb8ce6b1..9fff392b8 100644 --- a/backend/src/model/config/panel_api.rs +++ b/backend/src/model/config/panel_api.rs @@ -1,6 +1,5 @@ use crate::model::macros; -use shared::create_tuliprox_error_result; -use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{PanelApiConfigDto, PanelApiQueryParamDto, PanelApiQueryParametersDto}; #[derive(Debug, Clone)] @@ -66,24 +65,18 @@ impl PanelApiQueryParameters { .map(|p| p.value.trim().to_string()); match typ { Some(v) if v.eq_ignore_ascii_case("m3u") => Ok(()), - Some(v) => create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: unsupported type={v}, only m3u is supported"), - None => create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: missing required query param 'type=m3u'"), + Some(v) => info_err_res!("panel_api: unsupported type={v}, only m3u is supported"), + None => info_err_res!("panel_api: missing required query param 'type=m3u'"), } } fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); let Some(api_key) = api_key else { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} must contain query param 'api_key' (use value 'auto')" - ); + return info_err_res!("panel_api: {section} must contain query param 'api_key' (use value 'auto')"); }; if api_key.value.trim().is_empty() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" - ); + return info_err_res!("panel_api: {section} query param 'api_key' must not be empty (use value 'auto')"); } Ok(()) } @@ -92,18 +85,12 @@ impl PanelApiQueryParameters { let username = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("username")); let password = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("password")); if username.is_none() || password.is_none() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')" - ); + return info_err_res!("panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')"); } if !username.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) || !password.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)" - ); + return info_err_res!("panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)"); } Ok(()) } @@ -118,7 +105,7 @@ impl PanelApiQueryParameters { Self::require_api_key_param(params, "query_parameter.client_new")?; Self::validate_type_is_m3u(params)?; if params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("user")) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: client_new must not contain query param 'user'"); + return info_err_res!("panel_api: client_new must not contain query param 'user'"); } Ok(()) } @@ -176,19 +163,16 @@ impl PanelApiConfig { return Ok(()); } if self.url.trim().is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: url is missing"); + return info_err_res!("panel_api: url is missing"); } if self.api_key.as_ref().is_none_or(|k| k.trim().is_empty()) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "panel_api: api_key is missing"); + return info_err_res!("panel_api: api_key is missing"); } if self.query_parameter.client_info.is_empty() || self.query_parameter.client_new.is_empty() || self.query_parameter.client_renew.is_empty() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Info, - "panel_api: query_parameter.client_info/client_new/client_renew must be configured" - ); + return info_err_res!("panel_api: query_parameter.client_info/client_new/client_renew must be configured"); } self.query_parameter.prepare() } diff --git a/backend/src/model/config/source.rs b/backend/src/model/config/source.rs index ca73821e9..5272b3755 100644 --- a/backend/src/model/config/source.rs +++ b/backend/src/model/config/source.rs @@ -1,5 +1,5 @@ use crate::model::{macros, ConfigInput, ConfigTarget, ProcessTargets}; -use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, TuliproxError}; use shared::model::{ConfigSourceDto, PatternTemplate, SourcesConfigDto}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; @@ -67,7 +67,7 @@ impl TryFrom<&SourcesConfigDto> for SourcesConfig { // Validate that all input references exist for input_name in &source_dto.inputs { if !input_names.contains(input_name) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Source references unknown input: {input_name}"); + return info_err_res!("Source references unknown input: {input_name}"); } } sources.push(ConfigSource::from_dto(source_dto)?); @@ -140,7 +140,7 @@ impl SourcesConfig { let missing_targets: Vec = check_targets.iter().filter(|&(_, v)| *v == 0).map(|(k, _)| k.clone()).collect(); if !missing_targets.is_empty() { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "No target found for {}", missing_targets.join(", ")); + return info_err_res!("No target found for {}", missing_targets.join(", ")); } // let processing_targets: Vec = check_targets.iter().filter(|&(_, v)| *v != 0).map(|(k, _)| k.to_string()).collect(); // info!("Processing targets {}", processing_targets.join(", ")); diff --git a/backend/src/model/config/web_auth.rs b/backend/src/model/config/web_auth.rs index 6fb8083a9..192813a3f 100644 --- a/backend/src/model/config/web_auth.rs +++ b/backend/src/model/config/web_auth.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::BufRead; use std::path::PathBuf; -use shared::error::{TuliproxError, TuliproxErrorKind, create_tuliprox_error_result}; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{UserCredential, WebAuthConfigDto}; use crate::model::macros; use crate::utils; @@ -51,7 +51,7 @@ impl WebAuthConfig { if !utils::path_exists(&userfile_path) { userfile_path = PathBuf::from(config_path).join(&userfile_name); if !utils::path_exists(&userfile_path) { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Could not find userfile {}", &userfile_name); + return info_err_res!("Could not find userfile {}", &userfile_name); } } @@ -72,7 +72,7 @@ impl WebAuthConfig { self.t_users = Some(users); } else { - return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Could not read userfile {:?}", &userfile_path); + return info_err_res!("Could not read userfile {:?}", &userfile_path); } Ok(()) } diff --git a/backend/src/model/playlist.rs b/backend/src/model/playlist.rs index f20481bb5..e1a2204b7 100644 --- a/backend/src/model/playlist.rs +++ b/backend/src/model/playlist.rs @@ -1,21 +1,60 @@ -use crate::model::{TVGuide, ConfigInput}; -use shared::model::{PlaylistGroup}; +use std::collections::HashSet; +use crate::model::{ConfigInput, TVGuide}; +use shared::model::{PlaylistGroup, PlaylistItem, UUIDType}; +use crate::repository::PlaylistSource; -#[derive(Debug, Clone)] -pub struct FetchedPlaylist<'a> { // Contains playlist for one input +pub struct FetchedPlaylist<'a> { pub input: &'a ConfigInput, - pub playlist_groups: Vec, + pub source: Box, pub epg: Option, } impl FetchedPlaylist<'_> { - pub fn update_playlist(&mut self, plg: &PlaylistGroup) { - for grp in &mut self.playlist_groups { - if grp.id == plg.id { - plg.channels.iter().for_each(|item| grp.channels.push(item.clone())); - return; - } + pub fn sort_by_provider_ordinal(&mut self) { + self.source.sort_by_provider_ordinal(); + } + + pub async fn update_playlist(&mut self, plg: &PlaylistGroup) { + if self.source.is_memory() { + self.source.update_playlist(plg).await; } } + + pub fn is_memory(&self) -> bool { + self.source.is_memory() + } + + pub fn get_channel_count(&mut self) -> usize { + self.source.get_channel_count() + } + + pub fn get_group_count(&mut self) -> usize { + self.source.get_group_count() + } + + pub fn items_mut(&mut self) -> Box + Send + '_> { + self.source.items_mut() + } + + pub(crate) fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + self.source.items() + } + + pub fn get_missing_vod_info_count(&mut self) -> usize { + self.source.get_missing_vod_info_count() + } + + pub fn get_missing_series_info_count(&mut self) -> usize { + self.source.get_missing_series_info_count() + } + + pub fn deduplicate(&mut self, duplicates: &mut HashSet) { + self.source.deduplicate(duplicates); + } + + pub fn clone_source(&self) -> Box { + self.source.clone_box() + } } + diff --git a/backend/src/processing/mod.rs b/backend/src/processing/mod.rs index f9d8bc5c0..83cbd1f46 100644 --- a/backend/src/processing/mod.rs +++ b/backend/src/processing/mod.rs @@ -1,4 +1,5 @@ -mod playlist_watch; +pub(crate) mod input_cache; +pub(crate) mod playlist_watch; pub(crate) mod parser; pub(crate) mod processor; -pub(crate) mod input_cache; + diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 4dc42eb86..c12bf95c8 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -1,9 +1,10 @@ use crate::model::{Config, ConfigInput}; use crate::utils::request::DynReader; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use shared::utils::{default_supported_video_extensions, extract_id_from_url}; +use shared::utils::{default_supported_video_extensions, extract_id_from_url, StringInterner}; use std::borrow::BorrowMut; use tokio::io::AsyncBufReadExt; +use indexmap::IndexMap; // other implementations like calculating text_distance on all titles took too much time // we keep it now as simple as possible and less memory intensive. @@ -88,11 +89,11 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { } } -fn create_empty_playlistitem_header(input_name: &str, url: String) -> PlaylistItemHeader { +fn create_empty_playlistitem_header(interner: &mut StringInterner, input_name: &str, url: String) -> PlaylistItemHeader { PlaylistItemHeader { url, category_id: 0, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), ..Default::default() } } @@ -108,7 +109,7 @@ macro_rules! process_header_fields { }; } -fn process_header(input_name: &str, video_suffixes: &[String], content: &str, url: String) -> PlaylistItemHeader { +fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixes: &[String], content: &str, url: String) -> PlaylistItemHeader { let url_id = extract_id_from_url(&url); let url_types = if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { // TODO find Series based on group or configured names @@ -117,7 +118,7 @@ fn process_header(input_name: &str, video_suffixes: &[String], content: &str, ur None }; - let mut plih = create_empty_playlistitem_header(input_name, url); + let mut plih = create_empty_playlistitem_header(interner, input_name, url); let mut it = content.chars(); let mut stack = String::with_capacity(64); let line_token = token_till(&mut stack, &mut it, ':', false); @@ -147,14 +148,16 @@ fn process_header(input_name: &str, video_suffixes: &[String], content: &str, ur } else { process_header_fields!(plih, token.as_str(), (id, "tvg-id"), - (group, "group-title"), (name, "tvg-name"), (parent_code, "parent-code"), (audio_track, "audio-track"), (logo, "tvg-logo"), (logo_small, "tvg-logo-small"), (time_shift, "timeshift"), - (rec, "tvg-rec"); value); + (rec, "tvg-rec"); value.clone()); + if token == "group-title" { + plih.group = interner.intern(&value); + } } } } @@ -200,6 +203,7 @@ fn process_header(input_name: &str, video_suffixes: &[String], content: &str, ur pub async fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader, mut visit: F) { let mut header: Option = None; let mut group: Option = None; + let mut interner = StringInterner::new(); let input_name = input.name.as_str(); let video_suffixes = match cfg.video.as_ref() { @@ -209,6 +213,7 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp None => default_supported_video_extensions() }; let mut lines = tokio::io::BufReader::new(lines).lines(); + let mut ord_counter: u32 = 1; while let Ok(Some(line)) = lines.next_line().await { if line.starts_with("#EXTINF") { header = Some(line); @@ -222,14 +227,16 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp continue; } if let Some(header_value) = header { - let mut item = PlaylistItem { header: process_header(input_name, &video_suffixes, &header_value, line) }; + let mut item = PlaylistItem { header: process_header(&mut interner, input_name, &video_suffixes, &header_value, line) }; let header = &mut item.header; + header.source_ordinal = ord_counter; + ord_counter += 1; if header.group.is_empty() { if let Some(group_value) = group { - header.group = group_value; + header.group = interner.intern(&group_value); } else { let current_title = header.title.clone(); - header.group = get_title_group(current_title.as_str()); + header.group = interner.intern(&get_title_group(current_title.as_str())); } } visit(item); @@ -241,30 +248,17 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> Vec { - let mut sort_order: Vec> = vec![]; - let mut sort_order_idx: usize = 0; - let mut group_map: std::collections::HashMap = std::collections::HashMap::new(); + let mut group_map: IndexMap> = IndexMap::new(); consume_m3u(cfg, input, lines, |item| { - // keep the original sort order for groups and group the playlist items let key = { let header = &item.header; format!("{}{}", &header.xtream_cluster, &header.group) }; - match group_map.entry(key) { - std::collections::hash_map::Entry::Vacant(v) => { - v.insert(sort_order_idx); - sort_order.push(vec![item]); - sort_order_idx += 1; - } - std::collections::hash_map::Entry::Occupied(o) => { - if let Some(order) = sort_order.get_mut(*o.get()) { - order.push(item); - } - } - } + group_map.entry(key).or_default().push(item); }).await; + let mut grp_id = 0; - let result: Vec = sort_order.into_iter().filter_map(|channels| { + group_map.into_values().filter_map(|channels| { // create a group based on the first playlist item let channel = channels.first(); if let Some((cluster, group_title)) = channel.map(|pli| @@ -274,55 +268,58 @@ pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> V } else { None } - }).collect(); - result + }).collect() } #[cfg(test)] mod test { use crate::processing::parser::m3u::process_header; + use shared::utils::StringInterner; #[test] fn test_process_header_1() { + let mut interner = StringInterner::new(); let input: &str = "hello"; let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney" , Seven"#; - let pli = process_header(input, &video_suffixes, line, url.to_string()); + let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "abc-seven"); assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); assert_eq!(pli.chno, 7); - assert_eq!(pli.group, "Sydney"); + assert_eq!(&*pli.group, "Sydney"); } #[test] fn test_process_header_2() { + let mut interner = StringInterner::new(); let input: &str = "hello"; let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney", Seven"#; - let pli = process_header(input, &video_suffixes, line, url.to_string()); + let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "abc-seven"); assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); assert_eq!(pli.chno, 7); - assert_eq!(pli.group, "Sydney"); + assert_eq!(&*pli.group, "Sydney"); } #[test] fn test_process_header_xui_id() { + let mut interner = StringInterner::new(); let input: &str = "hello"; let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 tvg-id="abc-seven" xui-id="provider-123" group-title="Sydney", Seven"#; - let pli = process_header(input, &video_suffixes, line, url.to_string()); + let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "provider-123"); // Should use xui-id assert_eq!(pli.epg_channel_id, Some("abc-seven".to_string())); // Should preserve original tvg-id - assert_eq!(pli.group, "Sydney"); + assert_eq!(&*pli.group, "Sydney"); } } \ No newline at end of file diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 149265676..9a6650ec0 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -3,11 +3,17 @@ use crate::model::ConfigInput; use crate::model::{XtreamCategory}; use crate::utils::request::DynReader; use crate::utils::xtream::get_xtream_stream_url_base; -use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind}; -use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, XtreamCluster}; -use shared::utils::{generate_playlist_uuid, trim_last_slash}; +use shared::error::{notify_err_res, notify_err, TuliproxError}; +use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, + PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, + SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, + XtreamCluster, XtreamPlaylistItem}; +use shared::utils::{generate_playlist_uuid, trim_last_slash, StringInterner}; use indexmap::IndexMap; use tokio::task::spawn_blocking; +use serde::Deserializer; + + async fn map_to_xtream_category(categories: DynReader) -> Result, TuliproxError> { spawn_blocking(move || { @@ -15,10 +21,10 @@ async fn map_to_xtream_category(categories: DynReader) -> Result>(reader) { Ok(xtream_categories) => Ok(xtream_categories), Err(err) => { - create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process categories {}", &err) + notify_err_res!("Failed to process categories {}", &err) } } - }).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream categories failed: {e}")))? + }).await.map_err(|e| notify_err!("Mapping xtream categories failed: {e}"))? } async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader) -> Result, TuliproxError> { @@ -39,10 +45,10 @@ async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader Ok(stream_list) } Err(err) => { - create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {xtream_cluster}: {err}", ) + notify_err_res!("Failed to map to xtream streams {xtream_cluster}: {err}") } } - }).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream streams failed: {e}")))? + }).await.map_err(|e| notify_err!("Mapping xtream streams failed: {e}"))? } fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, password: &'a str, episode: &'a SeriesStreamDetailEpisodeProperties) -> Cow<'a, str> { @@ -55,7 +61,9 @@ fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, passwor } } -pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, group_title: &str, series_name: &str, input: &ConfigInput) -> Option> { + pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, + group_title: &str, series_name: &str, input: &ConfigInput, + interner: &mut StringInterner) -> Option> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); @@ -72,14 +80,14 @@ pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStre parent_code: parent_uuid.to_string(), name: series_name.to_string(), logo: episode.movie_image.clone(), - group: group_title.to_string(), + group: interner.intern(group_title), title: episode.title.clone(), url: episode_url.to_string(), item_type: PlaylistItemType::Series, xtream_cluster: XtreamCluster::Series, additional_properties: Some(StreamProperties::Episode(episode_info)), category_id: 0, - input_name: input.name.clone(), + input_name: interner.intern(&input.name), ..Default::default() } } @@ -129,6 +137,7 @@ pub async fn parse_xtream(input: &ConfigInput, streams: DynReader) -> Result>, TuliproxError> { match map_to_xtream_category(categories).await { Ok(xtream_categories) => { + let mut interner = StringInterner::new(); let input_name = input.name.clone(); let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); @@ -147,18 +156,20 @@ pub async fn parse_xtream(input: &ConfigInput, let (live_stream_use_prefix, live_stream_without_extension) = input.options.as_ref() .map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension)); + // Re-implement the loop to add ordinal + let mut ord_counter: u32 = 1; for stream in xtream_streams { let group = group_map.get_mut(&stream.get_category_id()).unwrap_or(&mut unknown_grp); let category_name = &group.category_name; let stream_url = create_xtream_url(xtream_cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); let item_type = PlaylistItemType::from(xtream_cluster); - let item = PlaylistItem { + let mut item = PlaylistItem { header: PlaylistItemHeader { id: stream.get_stream_id().to_string(), uuid: generate_playlist_uuid(&input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), name: stream.get_name().to_string(), logo: stream.get_stream_icon().to_string(), - group: category_name.clone(), + group: interner.intern(category_name), title: stream.get_name().to_string(), url: stream_url.to_string(), epg_channel_id: stream.get_epg_channel_id(), @@ -166,10 +177,12 @@ pub async fn parse_xtream(input: &ConfigInput, xtream_cluster, additional_properties: Some(stream), category_id: 0, - input_name: input_name.clone(), + input_name: interner.intern(&input_name), ..Default::default() }, }; + item.header.source_ordinal = ord_counter; + ord_counter += 1; group.add(item); } @@ -184,7 +197,7 @@ pub async fn parse_xtream(input: &ConfigInput, PlaylistGroup { id: category.category_id, xtream_cluster, - title: category.category_name.clone(), + title: interner.intern(&category.category_name), channels: category.channels.clone(), } }).collect())) @@ -196,6 +209,167 @@ pub async fn parse_xtream(input: &ConfigInput, } } +pub async fn parse_xtream_streaming( + input: &ConfigInput, + xtream_cluster: XtreamCluster, + categories: DynReader, + streams: DynReader, + mut on_item: F, +) -> Result, TuliproxError> +where + F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> + Send + 'static +{ + // 1. Parse Categories + let xtream_categories = map_to_xtream_category(categories).await?; + + // 2. Prepare for Stream Parsing + let mut interner = StringInterner::new(); + let input_name = input.name.clone(); + let url = input.url.as_str().to_string(); + let username = input.username.as_ref().map_or("", |v| v).to_string(); + let password = input.password.as_ref().map_or("", |v| v).to_string(); + let options = input.options.clone(); + + // Map categories for lookup + let group_map: IndexMap = xtream_categories.iter().map(|c| (c.category_id, c.category_name.clone())).collect(); + let unknown_group_name = "Unknown".to_string(); + + spawn_blocking(move || { + let reader = tokio_util::io::SyncIoBridge::new(streams); + let mut deserializer = serde_json::Deserializer::from_reader(reader); + + let (live_stream_use_prefix, live_stream_without_extension) = options.as_ref() + .map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension)); + + let mut source_ordinal = 0u32; + + match xtream_cluster { + XtreamCluster::Live => { + let mut on_stream = |stream: LiveStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Live(stream); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &mut interner, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + XtreamCluster::Video => { + let mut on_stream = |stream: VideoStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Video(Box::new(stream)); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &mut interner, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + XtreamCluster::Series => { + let mut on_stream = |stream: SeriesStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Series(Box::new(stream)); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &mut interner, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + } + Ok(()) + }).await.map_err(|e| notify_err!("Streaming parse failed: {e}"))??; + + Ok(xtream_categories) +} + +#[allow(clippy::too_many_arguments)] +fn process_stream_item( + input_name: &str, url: &str, username: &str, password: &str, + cluster: XtreamCluster, interner: &mut StringInterner, + group_map: &IndexMap, unknown_group_name: &str, + mut stream: StreamProperties, + callback: &mut F, + live_stream_use_prefix: bool, live_stream_without_extension: bool, + source_ordinal: u32, +) -> Result<(), TuliproxError> +where F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> +{ + stream.prepare(); + let category_id = stream.get_category_id(); + let category_name = group_map.get(&category_id).map_or(unknown_group_name, String::as_str); + let stream_url = create_xtream_url(cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); + + let item_type = PlaylistItemType::from(cluster); + let item = PlaylistItem { + header: PlaylistItemHeader { + id: stream.get_stream_id().to_string(), + uuid: generate_playlist_uuid(input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), + name: stream.get_name().to_string(), + logo: stream.get_stream_icon().to_string(), + group: interner.intern(category_name), + title: stream.get_name().to_string(), + url: stream_url.to_string(), + epg_channel_id: stream.get_epg_channel_id(), + item_type, + xtream_cluster: cluster, + additional_properties: Some(stream), + category_id, + source_ordinal, + input_name: interner.intern(input_name), + ..Default::default() + }, + }; + + // if let Some(StreamProperties::Series(props)) = item.header.additional_properties.as_mut() { + // // We need to set category_id for Series properties just like parse_xtream might expect or use? + // // Actually parse_xtream doesn't modify internal category_ids, but mapping to XtreamCategory struct relies on it. + // // Here we are creating PlaylistItem. + // let _ = props; + // } + + callback(XtreamPlaylistItem::from(&item)) +} + +struct XtreamItemVisitor<'a, T, F> { + on_item: &'a mut F, + _marker: std::marker::PhantomData, +} + +impl<'de, T, F> serde::de::Visitor<'de> for XtreamItemVisitor<'_, T, F> +where + T: serde::Deserialize<'de>, + F: FnMut(T) -> Result<(), TuliproxError>, +{ + type Value = (); + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a JSON array or an error object") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + while let Some(item) = seq.next_element::()? { + (self.on_item)(item).map_err(serde::de::Error::custom)?; + } + Ok(()) + } + + fn visit_map(self, map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let val: serde_json::Value = serde::de::Deserialize::deserialize(serde::de::value::MapAccessDeserializer::new(map))?; + if let Some(msg) = val.get("message").and_then(|m| m.as_str()) { + return Err(serde::de::Error::custom(format!("Xtream API error: {msg}"))); + } + Err(serde::de::Error::custom(format!("Expected array, got object: {val}"))) + } +} + #[cfg(test)] mod tests { use crate::processing::parser::xtream::map_to_xtream_streams; @@ -237,4 +411,20 @@ mod tests { } Ok(()) } -} \ No newline at end of file + + #[test] + fn test_xtream_item_visitor_array() { + use serde_json::Deserializer; + use shared::model::LiveStreamProperties; + let data = r#"[{"name":"stream1", "stream_id": 1, "category_id": 1, "added": "0"}]"#; + let mut deserializer = Deserializer::from_str(data); + let mut count = 0; + let mut on_item = |_: LiveStreamProperties| { + count += 1; + Ok(()) + }; + let visitor = super::XtreamItemVisitor { on_item: &mut on_item, _marker: std::marker::PhantomData }; + serde::Deserializer::deserialize_any(&mut deserializer, visitor).unwrap(); + assert_eq!(count, 1); + } +} diff --git a/backend/src/processing/processor/epg.rs b/backend/src/processing/processor/epg.rs index 400be91ed..98895cf80 100644 --- a/backend/src/processing/processor/epg.rs +++ b/backend/src/processing/processor/epg.rs @@ -1,8 +1,8 @@ use crate::model::{Epg, TVGuide, XmlTag, XmlTagIcon, EPG_ATTRIB_ID}; use crate::model::{EpgConfig, EpgSmartMatchConfig}; -use crate::model::{FetchedPlaylist}; +use crate::model::FetchedPlaylist; use crate::processing::parser::xmltv::normalize_channel_name; -use log::{debug, trace}; +use log::{debug, trace, warn}; use rphonetic::{DoubleMetaphone, Encoder}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; @@ -109,10 +109,12 @@ impl EpgIdCache<'_> { let smart_match_enabled = self.smart_match_enabled; let fuzzy_matching = self.fuzzy_match_enabled; - for channel in fp.playlist_groups.iter().flat_map(|g| &g.channels) { + // Helper closure to process a single item + // We use a closure here to capture `self` and avoid code duplication + let mut process_item = |name: &str, epg_channel_id: Option<&str>| { let mut missing_epg_id = true; // insert epg_id to known channel epg_ids - if let Some(id) = channel.header.epg_channel_id.as_deref() { + if let Some(id) = epg_channel_id { if !id.is_empty() { missing_epg_id = false; self.channel_epg_id.insert(Cow::Owned(id.to_string())); @@ -124,8 +126,13 @@ impl EpgIdCache<'_> { let needs_normalization = smart_match_enabled && (fuzzy_matching || missing_epg_id); if needs_normalization { - let name = &channel.header.name; - self.normalize_and_store(name, channel.header.epg_channel_id.as_deref()); + self.normalize_and_store(name, epg_channel_id); + } + }; + + for channel in fp.items() { + if channel.header.xtream_cluster == XtreamCluster::Live && channel.header.item_type.is_live() { + process_item(&channel.header.name, channel.header.epg_channel_id.as_deref()); } } } @@ -211,11 +218,13 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> } }; - let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live; - fp.playlist_groups.iter_mut() - .flat_map(|g| &mut g.channels) - .filter(filter_live) - .for_each(assign_values); + let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live && c.header.item_type.is_live(); + + if fp.is_memory() { + fp.items_mut().filter(filter_live).for_each(assign_values); + } else { + warn!("Disk based playlist modification is not supported!"); + } processed_epgs.push(epg_source); } } @@ -238,6 +247,9 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> /// process_playlist_epg(&mut playlist, &mut epg_data); /// ``` pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec) { + if fp.input.epg.is_none() { + return; + } // collect all epg_channel ids let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); id_cache.collect_epg_id(fp); diff --git a/backend/src/processing/processor/library.rs b/backend/src/processing/processor/library.rs index 5461062d4..734bc1c79 100644 --- a/backend/src/processing/processor/library.rs +++ b/backend/src/processing/processor/library.rs @@ -2,7 +2,7 @@ use crate::library::{MediaMetadata, MetadataAsyncIter, MetadataCacheEntry}; use crate::model::{AppConfig, ConfigInput}; use shared::error::TuliproxError; use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster}; -use shared::utils::{generate_playlist_uuid}; +use shared::utils::{generate_playlist_uuid, StringInterner}; use std::path::Path; use std::sync::Arc; @@ -25,15 +25,14 @@ pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &A channels: vec![], xtream_cluster: XtreamCluster::Series, }; + let mut interner = StringInterner::new(); while let Some(entry) = metadata_iter.next().await { match entry.metadata { MediaMetadata::Movie(_) => { - let pli = to_playlist_item(&entry, &input.name, &library_config.playlist.movie_category); - group_movies.channels.extend(pli); + to_playlist_item(&mut interner, &entry, &input.name, &library_config.playlist.movie_category, &mut group_movies.channels); } MediaMetadata::Series(_) => { - let pli = to_playlist_item(&entry, &input.name, &library_config.playlist.series_category); - group_series.channels.extend(pli); + to_playlist_item(&mut interner, &entry, &input.name, &library_config.playlist.series_category, &mut group_series.channels); } } } @@ -49,30 +48,30 @@ pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &A (groups, vec![]) } -fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &str) -> Vec { + +fn to_playlist_item(interner: &mut StringInterner, entry: &MetadataCacheEntry, input_name: &str, group_name: &str, channels: &mut Vec) { let metadata = &entry.metadata; match metadata { MediaMetadata::Movie(_) => { let additional_properties = metadata_cache_entry_to_xtream_movie_info(entry); - vec![PlaylistItem { + channels.push(PlaylistItem { header: PlaylistItemHeader { uuid: UUIDType::from_valid_uuid(&entry.uuid), name: metadata.title().to_string(), - group: group_name.to_string(), + group: interner.intern(group_name), title: metadata.title().to_string(), logo: metadata.poster().map_or_else(String::new, ToString::to_string), url: format!("file://{}", entry.file_path), xtream_cluster: XtreamCluster::Video, additional_properties, item_type: PlaylistItemType::LocalVideo, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), ..PlaylistItemHeader::default() } - }] + }); } MediaMetadata::Series(_series) => { - let mut items = vec![]; if let Some(additional_properties) = metadata_cache_entry_to_xtream_series_info(entry) { let mut episodes = vec![]; if let StreamProperties::Series(series_properties) = &additional_properties { @@ -92,13 +91,13 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s uuid: generate_playlist_uuid(input_name, &episode.id.to_string(), PlaylistItemType::LocalSeries, &episode.direct_source), logo: logo.clone(), name: episode.title.clone(), - group: group_name.to_string(), + group: interner.intern(group_name), title: episode.title.clone(), url: episode.direct_source.clone(), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeries, category_id: 0, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), additional_properties: Some(StreamProperties::Episode(EpisodeStreamProperties { episode_id: episode.id, episode: episode.episode_num, @@ -124,23 +123,20 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s uuid: UUIDType::from_valid_uuid(&entry.uuid), id: entry.uuid.clone(), name: metadata.title().to_string(), - group: group_name.to_string(), + group: interner.intern(group_name), title: metadata.title().to_string(), logo: metadata.poster().map_or_else(String::new, ToString::to_string), url: format!("file://{}", entry.file_path), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeriesInfo, - input_name: input_name.to_string(), + input_name: interner.intern(input_name), additional_properties: Some(additional_properties), ..PlaylistItemHeader::default() } }; - - items.push(series_info); - items.extend(episodes); + channels.push(series_info); + channels.extend(episodes); } - - items } } } diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 4fe2768c3..0d53e8203 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -1,16 +1,18 @@ -use crate::model::{AppConfig, Config, ConfigFavourites, ConfigInput, ConfigRename}; +use crate::model::{AppConfig, Config, ConfigFavourites, ConfigInput, ConfigRename, TVGuide}; use crate::utils::m3u; use crate::utils::xtream; use crate::utils::{epg, StepMeasureCallback}; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::Mutex; +use std::sync::{Arc, Weak}; +use tokio::sync::{Mutex, OwnedRwLockWriteGuard, RwLock}; use tokio::task::JoinSet; use crate::api::model::{EventManager, EventMessage, PlaylistStorageState, UpdateGuard}; use crate::messaging::send_message_json; use crate::model::Epg; + + use crate::model::FetchedPlaylist; use crate::model::Mapping; use crate::model::{ConfigTarget, ProcessTargets}; @@ -26,16 +28,20 @@ use crate::processing::processor::trakt::process_trakt_categories_for_target; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::{load_input_playlist, persist_input_playlist, persist_playlist}; +use crate::repository::xtream_repository::CategoryKey; +use crate::repository::{MemoryPlaylistSource, PlaylistSource}; use crate::utils::StepMeasure; use crate::utils::{debug_if_enabled, trace_if_enabled}; use futures::StreamExt; use indexmap::IndexMap; -use log::{debug, error, info, log_enabled, trace, warn, Level}; +use log::{debug, error, info, log_enabled, warn, Level}; use shared::error::{get_errors_notify_message, notify_err, TuliproxError}; use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider}; use shared::model::xtream_const::XTREAM_CLUSTER; -use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster}; -use shared::utils::{create_alias_uuid, default_as_default}; +use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, + PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState, + ProcessingOrder, UUIDType}; +use shared::utils::{create_alias_uuid, default_as_default, StringInterner}; use std::time::Instant; fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { @@ -43,14 +49,36 @@ fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { filter.filter(&provider) } -#[allow(clippy::unnecessary_wraps)] +pub fn apply_filter_to_source(source: &mut dyn PlaylistSource, filter: &Filter) -> Option> { + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); + for pli in source.into_items() { + if is_valid(&pli, filter, false) { + let group_title = pli.header.group.clone(); + let cluster = pli.header.xtream_cluster; + let cat_id = pli.header.category_id; + groups.entry(group_title.clone()) + .or_insert_with(|| PlaylistGroup { + id: cat_id, + title: group_title, + channels: vec![], + xtream_cluster: cluster, + }) + .channels.push(pli); + } + } + + if groups.is_empty() { None } else { Some(groups.into_values().collect()) } +} + +fn filter_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, _interner: &mut StringInterner) -> Option> { + apply_filter_to_source(source, &target.filter) +} + pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) -> Option> { - debug!("Filtering {} groups", playlist.len()); let mut new_playlist = Vec::with_capacity(128); for pg in playlist.iter_mut() { let channels = pg.channels.iter() .filter(|&pli| is_valid(pli, filter, false)).cloned().collect::>(); - trace!("Filtered group {} has now {}/{} items", pg.title, channels.len(), pg.channels.len()); if !channels.is_empty() { new_playlist.push(PlaylistGroup { id: pg.id, @@ -60,11 +88,7 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) }); } } - Some(new_playlist) -} - -fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { - apply_filter_to_playlist(playlist, &target.filter) + if new_playlist.is_empty() { None } else { Some(new_playlist) } } fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { @@ -86,7 +110,7 @@ fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { } } -fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>) { +fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>, interner: &mut StringInterner) { if let Some(renames) = rename { if !renames.is_empty() { let result = pli; @@ -97,33 +121,41 @@ fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>) { trace_if_enabled!("Renamed {}={value} to {cap}", &r.field); } let value = cap.into_owned(); - set_field_value(result, r.field, value); + set_field_value(result, r.field, value, interner); } } } } -fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { +fn rename_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, interner: &mut StringInterner) -> Option> { match &target.rename { - Some(renames) => { - if !renames.is_empty() { - let mut new_playlist: Vec = Vec::with_capacity(playlist.len()); - for g in playlist { - let mut grp = g.clone(); - for r in renames { - if matches!(r.field, ItemField::Group) { - let cap = r.pattern.replace_all(&grp.title, &r.new_name); - trace_if_enabled!("Renamed group {} to {cap} for {}", &grp.title, target.name); - grp.title = cap.into_owned(); + Some(renames) if !renames.is_empty() => { + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); + for mut pli in source.into_items() { + // Handle group rename first if it's in the renames + for r in renames { + if matches!(r.field, ItemField::Group) { + let value = &*pli.header.group; + let cap = r.pattern.replace_all(value, &r.new_name); + if *value != cap { + pli.header.group = interner.intern(&cap); } } - - grp.channels.iter_mut().for_each(|pli| exec_rename(pli, target.rename.as_ref())); - new_playlist.push(grp); } - return Some(new_playlist); + exec_rename(&mut pli, Some(renames), interner); + let group_title = pli.header.group.clone(); + let cluster = pli.header.xtream_cluster; + let cat_id = pli.header.category_id; + groups.entry(group_title.clone()) + .or_insert_with(|| PlaylistGroup { + id: cat_id, + title: group_title, + channels: vec![], + xtream_cluster: cluster, + }) + .channels.push(pli); } - None + Some(groups.into_values().collect()) } _ => None } @@ -164,41 +196,34 @@ fn map_channel_and_flatten(channel: PlaylistItem, mapping: &Mapping) -> Vec Option> { - if let Some(mappings) = target.mapping.load().as_ref() { - let new_playlist: Vec = playlist.iter().map(|playlist_group| { - let mut grp = playlist_group.clone(); - mappings.iter().filter(|&mapping| mapping.mapper.as_ref().is_some_and(|v| !v.is_empty())) - .for_each(|mapping| - grp.channels = grp.channels.drain(..).flat_map(|chan| map_channel_and_flatten(chan, mapping)).collect()); - grp - }).collect(); - - // if the group names are changed, restructure channels to the right groups - // we use - let mut new_groups: Vec = Vec::with_capacity(128); - let mut grp_id: u32 = 0; - for playlist_group in new_playlist { - for channel in &playlist_group.channels { - let cluster = &channel.header.xtream_cluster; - let title = &channel.header.group; - if let Some(grp) = new_groups.iter_mut().find(|x| *x.title == **title) { - grp.channels.push(channel.clone()); - } else { - grp_id += 1; - new_groups.push(PlaylistGroup { - id: grp_id, - title: title.clone(), - channels: vec![channel.clone()], - xtream_cluster: *cluster, - }); +fn map_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, _interner: &mut StringInterner) -> Option> { + let mapping_binding = target.mapping.load(); + let mappings = mapping_binding.as_ref()?; + let valid_mappings = mappings.iter().filter(|m| m.mapper.as_ref().is_some_and(|v| !v.is_empty())); + let iter: Box> = Box::new(source.into_items()); + let mapped_iter = valid_mappings.fold(iter, |iter, mapping| { + Box::new(iter.flat_map(move |chan| map_channel_and_flatten(chan, mapping))) + as Box> + }); + let mut next_groups: IndexMap, PlaylistGroup> = IndexMap::new(); + let mut grp_id: u32 = 0; + for channel in mapped_iter { + let group_title = channel.header.group.clone(); + let cluster = channel.header.xtream_cluster; + next_groups.entry(group_title.clone()) + .or_insert_with(|| { + grp_id += 1; + PlaylistGroup { + id: grp_id, + title: group_title, + channels: Vec::new(), + xtream_cluster: cluster, } - } - } - Some(new_groups) - } else { - None + }) + .channels.push(channel); } + + Some(next_groups.into_values().collect()) } fn map_playlist_counter(target: &ConfigTarget, playlist: &mut [PlaylistGroup]) { @@ -247,7 +272,7 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo (!user_targets.enabled && target.enabled) || (user_targets.enabled && user_targets.has_target(target.id)) } -async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc, input: &ConfigInput) -> (Vec, Vec, bool) { +async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc, input: &ConfigInput) -> (Vec, Vec, bool, bool) { let config = &*app_config.config.load(); let working_dir = &config.working_dir; @@ -286,14 +311,20 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc }; if fully_cached { - return (vec![], vec![], true); + return (vec![], vec![], true, false); } - let (playlist, errors) = match input.input_type { - InputType::M3u => m3u::download_m3u_playlist(client, config, input).await, + let (playlist, errors, persisted) = match input.input_type { + InputType::M3u => { + let (p, e) = m3u::download_m3u_playlist(client, config, input).await; + (p, e, false) + } InputType::Xtream => xtream::download_xtream_playlist(config, client, input, clusters_to_download.as_deref()).await, - InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![]), - InputType::Library => library::download_library_playlist(client, app_config, input).await, + InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![], false), + InputType::Library => { + let (p, e) = library::download_library_playlist(client, app_config, input).await; + (p, e, false) + } }; // Update Status @@ -330,20 +361,18 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc } } - (playlist, errors, false) + (playlist, errors, false, persisted) } -async fn process_source(client: &reqwest::Client, app_config: Arc, source_idx: usize, - user_targets: Arc, event_manager: Option>, - playlist_state: Option<&Arc>, -) -> (Vec, Vec, Vec) { - let sources = app_config.sources.load(); +async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (Vec, Vec, Vec) { + let sources = ctx.config.sources.load(); let mut errors = vec![]; let mut input_stats = HashMap::::new(); let mut target_stats = Vec::::new(); if let Some(source) = sources.get_source_at(source_idx) { + let mut interner = StringInterner::default(); let mut source_playlists = Vec::with_capacity(128); - let broadcast_step = create_broadcast_callback(event_manager.as_ref()); + let broadcast_step = create_broadcast_callback(ctx.event_manager.as_ref()); // Download the sources let mut source_downloaded = false; for input_name in &source.inputs { @@ -351,25 +380,15 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so error!("Input {input_name} referenced by source {source_idx} does not exist"); continue; }; - if is_input_enabled(input, &user_targets) { + if is_input_enabled(input, &ctx.user_targets) { source_downloaded = true; let start_time = Instant::now(); // Download the playlist for input - let (playlist_groups, mut error_list) = { + let (mut playlist_groups, mut error_list) = { broadcast_step("Playlist download", &format!("Downloading input '{}'", input.name)); - // Caching Logic - let (downloaded_playlist, mut download_err, was_cached) = playlist_download_from_input(client, &app_config, input).await; - let (playlist, error) = if was_cached { - match load_input_playlist(&app_config, input, None).await { - Ok(pl) => (pl, None), - Err(e) => (vec![], Some(e)), - } - } else { - broadcast_step("Playlist download", &format!("Persisting input '{}' playlist", input.name)); - persist_input_playlist(&app_config, input, downloaded_playlist).await - }; + let (mut download_err, playlist, error) = download_input(ctx, input).await; if let Some(err) = error { broadcast_step("Playlist download", &format!("Failed to persist/load input '{}' playlist", input.name)); @@ -379,60 +398,46 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so (playlist, download_err) }; - // Download epg for input - let (tvguide, mut tvguide_errors) = if error_list.is_empty() { - broadcast_step("Playlist download", &format!("Downloading epg for input '{}'", input.name)); - let working_dir = &app_config.config.load().working_dir; - epg::get_xmltv(client, input, working_dir).await + let (tvguide, mut tvguide_errors) = if input.input_type == InputType::Library { + (None, vec!()) } else { - (None, vec![]) + download_input_epg(ctx,input, &mut errors).await }; errors.append(&mut error_list); errors.append(&mut tvguide_errors); - let group_count = playlist_groups.len(); - let channel_count = playlist_groups.iter().map(|group| group.channels.len()).sum(); + let group_count = playlist_groups.get_group_count(); + let channel_count = playlist_groups.get_channel_count(); let input_name = &input.name; if playlist_groups.is_empty() { broadcast_step("Playlist download", &format!("Input '{}' playlist is empty", input.name)); info!("Source is empty {input_name}"); - errors.push(notify_err!(format!("Source is empty {input_name}"))); + errors.push(notify_err!("Source is empty {input_name}")); } else { source_playlists.push( FetchedPlaylist { input, - // If I create Arc here, it drops at end of loop iteration! - // SAFETY ISSUE. - // `source_playlists` stores `FetchedPlaylist`. - // `FetchedPlaylist` struct definition: - // pub struct FetchedPlaylist<'a> { pub input: &'a ConfigInput, ... } - // It holds a REFERENCE. - // If `input` comes from `sources` (guard), it lives as long as `sources` guard lives. - // `sources` guard is alive in this function scope. - // So `&input` is valid. - // `playlist_download_from_input` takes `&Arc`. I pass formatted Arc. - // `FetchedPlaylist` needs reference. - playlist_groups, + source: playlist_groups, epg: tvguide, } ); } let elapsed = start_time.elapsed().as_secs(); - input_stats.insert(input_name.clone(), create_input_stat(group_count, channel_count, error_list.len(), + input_stats.insert(input_name.clone(), create_input_stat(group_count, channel_count, errors.len(), input.input_type, input_name, elapsed)); } } if source_downloaded { if source_playlists.is_empty() { debug!("Source at index {source_idx} is empty"); - errors.push(notify_err!(format!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(std::string::String::as_str).collect::>().join(", ")))); + errors.push(notify_err!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(std::string::String::as_str).collect::>().join(", "))); } else { - debug_if_enabled!("Source has {} groups", source_playlists.iter().map(|fpl| fpl.playlist_groups.len()).sum::()); - let event_manager_clone = event_manager.clone(); + debug_if_enabled!("Source has {} groups", source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::()); for target in &source.targets { - let event_manager_clone = event_manager_clone.clone(); - if is_target_enabled(target, &user_targets) { - match process_playlist_for_target(&app_config, client, &mut source_playlists, target, &mut input_stats, &mut errors, event_manager_clone, playlist_state).await { + if is_target_enabled(target, &ctx.user_targets) { + match process_playlist_for_target(ctx, &mut source_playlists, target, + &mut input_stats, &mut errors, + &mut interner).await { Ok(()) => { target_stats.push(TargetStats::success(&target.name)); } @@ -449,6 +454,58 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so (input_stats.into_values().collect(), target_stats, errors) } +async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc, + error_list: &mut [TuliproxError]) -> (Option, Vec) { + // Download epg for input + let (tvguide, tvguide_errors) = if error_list.is_empty() { + debug!("Downloading epg for input '{}'", input.name); + let working_dir = &ctx.config.config.load().working_dir; + epg::get_xmltv(ctx, input, working_dir).await + } else { + (None, vec![]) + }; + (tvguide, tvguide_errors) +} + +async fn download_input(ctx: &PlaylistProcessingContext, input: &Arc) + -> (Vec, Box, Option) { + // Coordination Logic + let need_download = !ctx.is_input_downloaded(&input.name).await; + + let (downloaded_playlist, download_err, was_cached, persisted) = if need_download { + // Acquire named lock to prevent thundering herd on same input + let _input_lock = ctx.get_input_lock(&input.name).await; + // Check again after lock + let already_processed = ctx.is_input_downloaded(&input.name).await; + + if already_processed { + // Use empty results, will load from disk below + (vec![], vec![], true, false) + } else { + let res = playlist_download_from_input(&ctx.client, &ctx.config, input).await; + // Mark as processed if NO critical errors? + // playlist_download_from_input returns errors but also potentially a partial playlist. + // If it attempted download, we consider it processed for this session. + ctx.mark_input_downloaded(input.name.clone()).await; + res + } + } else { + (vec![], vec![], true, false) + }; + + let (playlist, error) = if was_cached || persisted { + match load_input_playlist(ctx, input, None).await { + Ok(pl_source) => (pl_source, None), + Err(e) => (MemoryPlaylistSource::default().boxed(), Some(e)), + } + } else { + debug!("Persisting input '{}' playlist", input.name); + let (pl, err) = persist_input_playlist(&ctx.config, input, downloaded_playlist).await; + (MemoryPlaylistSource::new(pl).boxed(), err) + }; + (download_err, playlist, error) +} + fn create_broadcast_callback(event_manager: Option<&Arc>) -> StepMeasureCallback { if let Some(event_mgr) = event_manager { let events = event_mgr.clone(); @@ -477,42 +534,80 @@ fn create_input_stat(group_count: usize, channel_count: usize, error_count: usiz } } -async fn process_sources(client: &reqwest::Client, config: &Arc, user_targets: Arc, - event_manager: Option>, playlist_state: Option<&Arc>, -) -> (Vec, Vec) { + +#[derive(Clone)] +pub struct PlaylistProcessingContext { + pub client: reqwest::Client, + pub config: Arc, + pub user_targets: Arc, + pub event_manager: Option>, + pub playlist_state: Option>, + + // Coordination + processed_inputs: Arc>>, + input_locks: Arc>>>>, +} + +impl PlaylistProcessingContext { + pub async fn is_input_downloaded(&self, input_name: &str) -> bool { + let processed = self.processed_inputs.lock().await; + processed.contains(input_name) + } + pub async fn mark_input_downloaded(&self, input_name: String) -> bool { + let mut processed = self.processed_inputs.lock().await; + processed.insert(input_name) + } + + pub async fn get_input_lock(&self, input_name: &str) -> OwnedRwLockWriteGuard<()> { + let mut locks = self.input_locks.lock().await; + // Clean up stale weak references + locks.retain(|_, weak| weak.strong_count() > 0); + + if let Some(weak) = locks.get(input_name) { + if let Some(strong) = weak.upgrade() { + return strong.write_owned().await; + } + } + + let lock = Arc::new(RwLock::new(())); + locks.insert(input_name.to_string(), Arc::downgrade(&lock)); + lock.write_owned().await + } +} + +async fn process_sources(processing_ctx: &PlaylistProcessingContext) -> (Vec, Vec) { let mut async_tasks = JoinSet::new(); - let sources = config.sources.load(); - let process_parallel = config.config.load().process_parallel && sources.sources.len() > 1; + let sources = processing_ctx.config.sources.load(); + let process_parallel = processing_ctx.config.config.load().process_parallel && sources.sources.len() > 1; if process_parallel && log_enabled!(Level::Debug) { debug!("Parallel processing enabled"); } + let errors = Arc::new(Mutex::>::new(vec![])); let stats = Arc::new(Mutex::>::new(vec![])); + for (index, source) in sources.sources.iter().enumerate() { - if !source.should_process_for_user_targets(&user_targets) { + if !source.should_process_for_user_targets(&processing_ctx.user_targets) { continue; } // We're using the file lock this way on purpose let source_lock_path = PathBuf::from(format!("source_{index}")); - let Ok(update_lock) = config.file_locks.try_write_lock(&source_lock_path).await else { + let Ok(update_lock) = processing_ctx.config.file_locks.try_write_lock(&source_lock_path).await else { warn!("The update operation for the source at index {index} was skipped because an update is already in progress."); continue; }; let shared_errors = errors.clone(); let shared_stats = stats.clone(); - let cfg = config.clone(); - let usr_trgts = user_targets.clone(); - let event_manager = event_manager.clone(); + let ctx = processing_ctx.clone(); + if process_parallel { - let http_client = client.clone(); - let playlist_state = playlist_state.cloned(); async_tasks.spawn(async move { // Hold the per-source lock for the full duration of this update. let current_update_lock = update_lock; let (input_stats, target_stats, mut res_errors) = - process_source(&http_client, cfg, index, usr_trgts, event_manager, playlist_state.as_ref()).await; + process_source(index, &ctx).await; shared_errors.lock().await.append(&mut res_errors); if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) { shared_stats.lock().await.push(process_stats); @@ -521,7 +616,7 @@ async fn process_sources(client: &reqwest::Client, config: &Arc, user }); } else { let (input_stats, target_stats, mut res_errors) = - process_source(client, cfg, index, usr_trgts, event_manager, playlist_state).await; + process_source(index, &ctx).await; shared_errors.lock().await.append(&mut res_errors); if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) { shared_stats.lock().await.push(process_stats); @@ -541,7 +636,7 @@ async fn process_sources(client: &reqwest::Client, config: &Arc, user } } -pub type ProcessingPipe = Vec Option>>; +pub type ProcessingPipe = Vec Option>>; fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { match &target.processing_order { @@ -554,29 +649,27 @@ fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { } } -fn duplicate_hash(item: &PlaylistItem) -> UUIDType { - item.get_uuid() -} - fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>, - duplicates: &mut HashSet) -> FetchedPlaylist<'a> { + duplicates: &mut HashSet, + interner: &mut StringInterner) -> FetchedPlaylist<'a> { let mut new_fpl = FetchedPlaylist { input: fpl.input, - playlist_groups: fpl.playlist_groups.clone(), // we need to clone, because of multiple target definitions, we cant change the initial playlist. + source: fpl.clone_source(), epg: fpl.epg.clone(), }; if target.options.as_ref().is_some_and(|opt| opt.remove_duplicates) { - for group in &mut new_fpl.playlist_groups { - // `HashSet::insert` returns true for first insert, otherwise false - group.channels.retain(|item| duplicates.insert(duplicate_hash(item))); - } + new_fpl.deduplicate(duplicates); } for f in pipe { - if let Some(groups) = f(&mut new_fpl.playlist_groups, target) { - new_fpl.playlist_groups = groups; + if let Some(groups) = f(new_fpl.source.as_mut(), target, interner) { + new_fpl.source = MemoryPlaylistSource::new(groups).boxed(); } } + // Ensure source is memory-based for downstream mutable processing (VOD/series resolution) + if !new_fpl.is_memory() { + new_fpl.source = MemoryPlaylistSource::new(new_fpl.source.take_groups()).boxed(); + } new_fpl } @@ -585,9 +678,9 @@ fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedP fn flatten_groups(playlistgroups: Vec) -> Vec { let mut sort_order: Vec = vec![]; let mut idx: usize = 0; - let mut group_map: HashMap<(String, XtreamCluster), usize> = HashMap::new(); + let mut group_map: HashMap = HashMap::new(); for group in playlistgroups { - let key = (group.title.clone(), group.xtream_cluster); + let key = (group.xtream_cluster, group.title.clone()); match group_map.entry(key) { std::collections::hash_map::Entry::Vacant(v) => { v.insert(idx); @@ -605,14 +698,12 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { } #[allow(clippy::too_many_arguments)] -async fn process_playlist_for_target(app_config: &Arc, - client: &reqwest::Client, +async fn process_playlist_for_target(ctx: &PlaylistProcessingContext, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, stats: &mut HashMap, errors: &mut Vec, - event_manager: Option>, - playlist_state: Option<&Arc>, + interner: &mut StringInterner, ) -> Result<(), Vec> { debug_if_enabled!("Processing order is {}", &target.processing_order); @@ -620,22 +711,23 @@ async fn process_playlist_for_target(app_config: &Arc, let mut processed_fetched_playlists: Vec = vec![]; debug!("Executing processing pipes"); - let broadcast_step = create_broadcast_callback(event_manager.as_ref()); + let broadcast_step = create_broadcast_callback(ctx.event_manager.as_ref()); let pipe = get_processing_pipe(target); let mut step = StepMeasure::new(&target.name, broadcast_step); for provider_fpl in playlists.iter_mut() { step.broadcast("Executing transformations on '{}' playlist", &target.name); - let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); - playlist_resolve_series(app_config, client, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; - playlist_resolve_vod(app_config, client, target, errors, &mut processed_fpl).await; + let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates, interner); + processed_fpl.sort_by_provider_ordinal(); + playlist_resolve_series(&ctx.config, &ctx.client, target, errors, &pipe, provider_fpl, &mut processed_fpl, interner).await; + playlist_resolve_vod(&ctx.config, &ctx.client, target, errors, provider_fpl, &mut processed_fpl).await; // stats - let input_stats = stats.get_mut(&processed_fpl.input.name); - if let Some(stat) = input_stats { - stat.processed_stats.group_count = processed_fpl.playlist_groups.len(); - stat.processed_stats.channel_count = processed_fpl.playlist_groups.iter() - .map(|group| group.channels.len()) - .sum(); + let input_entry_name = processed_fpl.input.name.clone(); + let group_count = processed_fpl.get_group_count(); + let channel_count = processed_fpl.get_channel_count(); + if let Some(stat) = stats.get_mut(&input_entry_name) { + stat.processed_stats.group_count = group_count; + stat.processed_stats.channel_count = channel_count; } processed_fetched_playlists.push(processed_fpl); } @@ -654,7 +746,7 @@ async fn process_playlist_for_target(app_config: &Arc, Ok(()) } else { // Process Trakt categories - if trakt_playlist(client, target, errors, &mut new_playlist).await { + if trakt_playlist(&ctx.client, target, errors, &mut new_playlist).await { step.tick("trakt categories"); } @@ -669,11 +761,11 @@ async fn process_playlist_for_target(app_config: &Arc, map_playlist_counter(target, &mut flat_new_playlist); step.tick("assigning channel counter"); - let config = app_config.config.load(); - if process_watch(&config, client, target, &flat_new_playlist).await { + let config = ctx.config.config.load(); + if process_watch(&config, &ctx.client, target, &flat_new_playlist).await { step.tick("group watches"); } - let result = persist_playlist(app_config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, playlist_state).await; + let result = persist_playlist(&ctx.config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, ctx.playlist_state.as_ref(), interner).await; step.stop("Persisting playlists"); result } @@ -681,7 +773,7 @@ async fn process_playlist_for_target(app_config: &Arc, pub fn process_favourites(playlist: &mut Vec, favourites_cfg: Option<&[ConfigFavourites]>) { if let Some(favourites) = favourites_cfg { - let mut fav_groups: IndexMap> = IndexMap::new(); + let mut fav_groups: IndexMap, Vec> = IndexMap::new(); for pg in playlist.iter() { for pli in &pg.channels { // series episodes cant be included in favourites @@ -737,14 +829,14 @@ async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors: } async fn process_epg(processed_fetched_playlists: &mut Vec>) -> (Vec, Vec) { - let mut new_playlist = vec![]; + let mut new_playlist: Vec = vec![]; let mut new_epg = vec![]; // each fetched playlist can have its own epgl url. // we need to process each input epg. for fp in processed_fetched_playlists { process_playlist_epg(fp, &mut new_epg).await; - new_playlist.append(&mut fp.playlist_groups); + new_playlist.extend(fp.source.take_groups()); } (new_epg, new_playlist) } @@ -786,9 +878,19 @@ pub async fn exec_processing(client: &reqwest::Client, app_config: Arc Ordering { - let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.clone() }; - let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.clone() }; + let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.to_string() }; + let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.to_string() }; playlist_comparator(group_sort.sequence.as_ref(), group_sort.order, &value_a, &value_b) } @@ -144,7 +144,7 @@ pub(in crate::processing::processor) fn sort_playlist(target: &ConfigTarget, new } let regexp = &channel_sort.group_pattern; for group in new_playlist.iter_mut() { - let group_title = if match_as_ascii { deunicode(&group.title) } else { group.title.clone() }; + let group_title = if match_as_ascii { deunicode(&group.title) } else { group.title.to_string() }; if regexp.is_match(group_title.as_str()) { group.channels.sort_by(|chan1, chan2| playlistitem_comparator(chan1, chan2, channel_sort, match_as_ascii)); } diff --git a/backend/src/processing/processor/trakt.rs b/backend/src/processing/processor/trakt.rs index b68873355..f4353338f 100644 --- a/backend/src/processing/processor/trakt.rs +++ b/backend/src/processing/processor/trakt.rs @@ -4,6 +4,7 @@ use crate::utils::{extract_year_from_title, normalize_title_for_matching, TraktC use crate::utils::{trace_if_enabled, with}; use log::{debug, info, trace, warn}; use shared::error::TuliproxError; +use shared::utils::StringInterner; use shared::model::{FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, TraktContentType, XtreamCluster}; use shared::utils::CONSTANTS; use std::borrow::Cow; @@ -128,6 +129,7 @@ fn find_best_match_for_item<'a>( fn create_category_from_matches<'a>( matches: Vec>, list_config: &'a TraktListConfig, + interner: &mut StringInterner, ) -> Option { if matches.is_empty() { return None; } @@ -164,7 +166,7 @@ fn create_category_from_matches<'a>( header.set_field("caption", &caption); } } - header.group = String::from(group_title); + header.group = interner.intern(group_title); header.gen_uuid(); }); matched_items.push(modified_item); @@ -184,7 +186,7 @@ fn create_category_from_matches<'a>( Some(PlaylistGroup { id: 0, - title: String::from(group_title), + title: interner.intern(group_title), channels: matched_items, xtream_cluster: cluster, }) @@ -194,6 +196,7 @@ fn match_trakt_items_with_playlist<'a>( trakt_items: &'a [TraktListItem], playlist: &'a [PlaylistGroup], list_config: &'a TraktListConfig, + interner: &mut StringInterner, ) -> Option { let trakt_match_items: Vec> = trakt_items .iter() @@ -217,7 +220,7 @@ fn match_trakt_items_with_playlist<'a>( } } - create_category_from_matches(matches, list_config) + create_category_from_matches(matches, list_config, interner) } pub struct TraktCategoriesProcessor { @@ -244,6 +247,8 @@ impl TraktCategoriesProcessor { info!("Processing {} Trakt lists for target {}", trakt_config.lists.len(), target.name); let mut new_categories = Vec::new(); let mut total_matches = 0; + let mut interner = StringInterner::new(); + for list_config in &trakt_config.lists { let cache_key = format!("{}:{}", list_config.user, list_config.list_slug); @@ -251,7 +256,7 @@ impl TraktCategoriesProcessor { Ok(trakt_items) => { debug!("Processing Trakt list {cache_key} with {} items", trakt_items.len()); - if let Some(category) = match_trakt_items_with_playlist(&trakt_items, playlist, list_config) { + if let Some(category) = match_trakt_items_with_playlist(&trakt_items, playlist, list_config, &mut interner) { if !category.channels.is_empty() { total_matches += category.channels.len(); let category_len = category.channels.len(); diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index 331a93e4a..911bfc172 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -9,10 +9,10 @@ pub(in crate::processing) async fn playlist_resolve_download_playlist_item(clien let provider_id = pli.get_provider_id()?; if let Some(info_url) = xtream::get_xtream_player_api_info_url(input, cluster, provider_id) { let input_source = InputSource::from(input).with_url(info_url); - result = match xtream::get_xtream_stream_info_content(client, &input_source).await { + result = match xtream::get_xtream_stream_info_content(client, &input_source, true).await { Ok(content) => Some(content), Err(err) => { - errors.push(info_err!(format!("{err}"))); + errors.push(info_err!("{err}")); None } }; diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index b7f1fa887..3a10d34f1 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -5,21 +5,28 @@ use crate::processing::processor::create_resolve_options_function_for_xtream_tar use crate::processing::processor::playlist::ProcessingPipe; use crate::processing::processor::xtream::playlist_resolve_download_playlist_item; use crate::repository::storage::get_input_storage_path; -use crate::repository::xtream_repository::persists_input_series_info; +use crate::repository::xtream_repository::persist_input_series_info_batch; +use crate::repository::{MemoryPlaylistSource, PlaylistSource}; use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; use shared::model::{InputType, PlaylistEntry, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo}; use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster}; +use shared::utils::StringInterner; +use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; create_resolve_options_function_for_xtream_target!(series); +const BATCH_SIZE: usize = 100; + +#[allow(clippy::too_many_lines)] async fn playlist_resolve_series_info(app_config: &Arc, client: &reqwest::Client, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_series: bool, - resolve_delay: u16) -> Vec { + resolve_delay: u16, + interner: &mut StringInterner) -> Vec { let input = fpl.input; let working_dir = &app_config.config.load().working_dir; let storage_path = match get_input_storage_path(&input.name, working_dir) { @@ -30,88 +37,96 @@ async fn playlist_resolve_series_info(app_config: &Arc, client: &reqw } }; - let series_info_count = if resolve_series { - let count = fpl.playlist_groups.iter() - .flat_map(|plg| &plg.channels) - .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Series - && pli.header.item_type == PlaylistItemType::SeriesInfo - && !pli.has_details()).count(); - info!("Found {count} series info to resolve"); - count - } else { - 0 - }; + let series_info_count = fpl.get_missing_series_info_count(); + + if series_info_count > 0 { + info!("Found {series_info_count} series info to resolve"); + } let mut last_log_time = Instant::now(); let mut processed_series_info_count = 0; - let mut result: Vec = vec![]; + let mut group_series: HashMap = HashMap::new(); + let mut batch = Vec::with_capacity(BATCH_SIZE); - for plg in &mut fpl.playlist_groups { - let mut group_series = vec![]; - for pli in &mut plg.channels { - if pli.header.xtream_cluster != XtreamCluster::Series - || pli.header.item_type != PlaylistItemType::SeriesInfo { - continue; - } + let input = fpl.input; + for pli in fpl.items_mut() { + if pli.header.xtream_cluster != XtreamCluster::Series + || pli.header.item_type != PlaylistItemType::SeriesInfo { + continue; + } - let Some(provider_id) = pli.get_provider_id() else { continue; }; - if provider_id == 0 { - continue; - } + let Some(provider_id) = pli.get_provider_id() else { continue; }; + if provider_id == 0 { + continue; + } - let should_download = resolve_series && !pli.has_details(); - if should_download { - processed_series_info_count += 1; - if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await { - if !content.is_empty() { - match serde_json::from_str::(&content) { - Ok(info) => { - let series_stream_props = SeriesStreamProperties::from_info(&info, pli); - // the input db needs to be updated - let _ = persists_input_series_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &series_stream_props).await; - // Update in-memory playlist items with the newly fetched vod info. - // This makes the data available for later processing steps like STRM export. - pli.header.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props))); - } - Err(err) => { - error!("Failed to parse series info for provider_id {provider_id}: {err}"); + let should_download = resolve_series && !pli.has_details(); + if should_download { + processed_series_info_count += 1; + if let Some(content) = playlist_resolve_download_playlist_item(client, pli, input, errors, resolve_delay, XtreamCluster::Series).await { + if !content.is_empty() { + match serde_json::from_str::(&content) { + Ok(info) => { + let series_stream_props = SeriesStreamProperties::from_info(&info, pli); + + batch.push((provider_id, series_stream_props.clone())); + if batch.len() >= BATCH_SIZE { + if let Err(err) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, &input.name, std::mem::take(&mut batch)).await { + error!("Failed to persist batch series info: {err}"); + } } + + // Update in-memory playlist items with the newly fetched vod info. + // This makes the data available for later processing steps like STRM export. + pli.header.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props))); + } + Err(err) => { + error!("Failed to parse series info for provider_id {provider_id}: {err}"); } } } } + } - // extract episodes from info - if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() { - let (group, series_name) = { - let header = &pli.header; - (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) - }; - if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input) { - group_series.extend(episodes.into_iter()); - } - } - - if resolve_series && log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { - info!("resolved {processed_series_info_count}/{series_info_count} series info"); - last_log_time = Instant::now(); + // extract episodes from info + if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() { + let (group, series_name) = { + let header = &pli.header; + (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) + }; + if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input, interner) { + let group = group_series.entry(pli.header.category_id) + .or_insert_with(|| { + PlaylistGroup { + id: pli.header.category_id, + title: pli.header.group.clone(), + channels: Vec::new(), + xtream_cluster: XtreamCluster::Series, + } + }); + group.channels.extend(episodes.into_iter()); } } - if !group_series.is_empty() { - result.push(PlaylistGroup { - id: plg.id, - title: plg.title.clone(), - channels: group_series, - xtream_cluster: XtreamCluster::Series, - }); + + if resolve_series && log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_series_info_count}/{series_info_count} series info"); + last_log_time = Instant::now(); } } + + if !batch.is_empty() { + if let Err(err) = persist_input_series_info_batch(app_config, &storage_path, XtreamCluster::Series, &input.name, batch).await { + error!("Failed to persist final batch series info: {err}"); + } + } + if resolve_series { info!("resolved {processed_series_info_count}/{series_info_count} series info"); } - result + group_series.into_values().collect() } +#[allow(clippy::too_many_arguments)] pub async fn playlist_resolve_series(cfg: &Arc, client: &reqwest::Client, target: &ConfigTarget, @@ -119,25 +134,34 @@ pub async fn playlist_resolve_series(cfg: &Arc, pipe: &ProcessingPipe, provider_fpl: &mut FetchedPlaylist<'_>, processed_fpl: &mut FetchedPlaylist<'_>, + interner: &mut StringInterner, ) { let (resolve_series, resolve_delay) = get_resolve_series_options(target, processed_fpl); - let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay).await; + provider_fpl.source.release_resources(XtreamCluster::Series); + let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay, interner).await; + provider_fpl.source.obtain_resources().await; if series_playlist.is_empty() { return; } - // original content saved into original list - for plg in &series_playlist { - provider_fpl.update_playlist(plg); - } - // run processing pipe over new items - let mut new_playlist = series_playlist; - for f in pipe { - if let Some(v) = f(&mut new_playlist, target) { - new_playlist = v; + if provider_fpl.is_memory() { + // original content saved into original list + for plg in &series_playlist { + provider_fpl.update_playlist(plg).await; } } + // run the processing pipe over new items + let mut new_playlist = series_playlist; + for f in pipe { + let mut source = MemoryPlaylistSource::new(new_playlist); + if let Some(v) = f(&mut source, target, interner) { + new_playlist = v; + } else { + new_playlist = source.take_groups(); + } + } + // assign new items to the new playlist for plg in &new_playlist { - processed_fpl.update_playlist(plg); + processed_fpl.update_playlist(plg).await; } } diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 76a7e7fe8..2b6b792ee 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -3,7 +3,7 @@ use crate::model::{AppConfig, ConfigTarget}; use crate::processing::processor::create_resolve_options_function_for_xtream_target; use crate::processing::processor::xtream::playlist_resolve_download_playlist_item; use crate::repository::storage::get_input_storage_path; -use crate::repository::xtream_repository::persist_input_vod_info; +use crate::repository::xtream_repository::persist_input_vod_info_batch; use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; use shared::model::{InputType, PlaylistEntry, StreamProperties, VideoStreamProperties, XtreamVideoInfo}; @@ -11,10 +11,16 @@ use shared::model::{PlaylistItemType, XtreamCluster}; use std::sync::Arc; use std::time::Instant; + create_resolve_options_function_for_xtream_target!(vod); -pub async fn playlist_resolve_vod(app_config: &Arc, client: &reqwest::Client, - target: &ConfigTarget, errors: &mut Vec, +const BATCH_SIZE: usize = 100; + +pub async fn playlist_resolve_vod(app_config: &Arc, + client: &reqwest::Client, + target: &ConfigTarget, + errors: &mut Vec, + provider_fpl: &mut FetchedPlaylist<'_>, fpl: &mut FetchedPlaylist<'_>) { let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl); if !resolve_movies { return; } @@ -29,50 +35,59 @@ pub async fn playlist_resolve_vod(app_config: &Arc, client: &reqwest: } }; - // LocalVideo entries are not resolved! - let vod_info_count = fpl.playlist_groups.iter() - .flat_map(|plg| &plg.channels) - .filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video - && pli.header.item_type == PlaylistItemType::Video - && !pli.has_details()).count(); + let vod_info_count = fpl.get_missing_vod_info_count(); + info!("Found missing {vod_info_count} vod info to resolve"); - - info!("Found {vod_info_count} vod info to resolve"); let mut last_log_time = Instant::now(); let mut processed_vod_info_count = 0; + let mut batch = Vec::with_capacity(BATCH_SIZE); - for plg in &mut fpl.playlist_groups { - for pli in &mut plg.channels { - if pli.header.xtream_cluster != XtreamCluster::Video - || pli.header.item_type != PlaylistItemType::Video - || pli.has_details() { - continue; - } - let Some(provider_id) = pli.get_provider_id() else { continue; }; - processed_vod_info_count += 1; - if provider_id != 0 { - if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await { - if content.is_empty() { continue; } - match serde_json::from_str::(&content) { - Ok(info) => { - let video_stream_props = VideoStreamProperties::from_info(&info, pli); - if let Err(err) = persist_input_vod_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &video_stream_props).await { - error!("Failed to persist VOD info for provider_id {provider_id}: {err}"); + provider_fpl.source.release_resources(XtreamCluster::Video); + + let input = fpl.input; + for pli in fpl.items_mut() { + if pli.header.xtream_cluster != XtreamCluster::Video + || pli.header.item_type != PlaylistItemType::Video + || pli.has_details() { + continue; + } + let Some(provider_id) = pli.get_provider_id() else { continue; }; + processed_vod_info_count += 1; + if provider_id != 0 { + if let Some(content) = playlist_resolve_download_playlist_item(client, pli, input, errors, resolve_delay, XtreamCluster::Video).await { + if content.is_empty() { continue; } + match serde_json::from_str::(&content) { + Ok(info) => { + let video_stream_props = VideoStreamProperties::from_info(&info, pli); + + batch.push((provider_id, video_stream_props.clone())); + if batch.len() >= BATCH_SIZE { + if let Err(err) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, &input.name, std::mem::take(&mut batch)).await { + error!("Failed to persist batch VOD info: {err}"); } - // This makes the data available for subsequent processing steps like STRM export. - pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props))); - } - Err(err) => { - error!("Failed to parse video info for provider_id {provider_id}: {err}"); } + + // This makes the data available for subsequent processing steps like STRM export. + pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props))); + } + Err(err) => { + error!("Failed to parse video info for provider {} stream_id {provider_id}: {err} {content}", input.name); } } } - if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { - info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); - last_log_time = Instant::now(); - } + } + if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); + last_log_time = Instant::now(); } } + + if !batch.is_empty() { + if let Err(err) = persist_input_vod_info_batch(app_config, &storage_path, XtreamCluster::Video, &input.name, batch).await { + error!("Failed to persist final batch VOD info: {err}"); + } + } + + provider_fpl.source.obtain_resources().await; info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); } diff --git a/backend/src/repository/bplustree.rs b/backend/src/repository/bplustree.rs index 7dbbc4312..d3c4a9852 100644 --- a/backend/src/repository/bplustree.rs +++ b/backend/src/repository/bplustree.rs @@ -1,17 +1,17 @@ -use std::ffi::OsString; use crate::utils; use crate::utils::{binary_deserialize, binary_serialize}; +use fs2::FileExt; use indexmap::IndexMap; use log::error; use serde::{Deserialize, Serialize}; use shared::error::{string_to_io_error, to_io_error}; +use std::ffi::OsString; use std::fs::{File, OpenOptions}; use std::io::{self, BufReader, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; use std::mem::size_of; use std::path::{Path, PathBuf}; use tempfile::NamedTempFile; -use fs2::FileExt; // Constants (Restored) const PAGE_SIZE: u16 = 4096; @@ -231,7 +231,6 @@ impl From for BPlusTreeError { } - impl BPlusTreeError { pub fn to_io(self) -> io::Error { match self { @@ -255,7 +254,6 @@ impl From for BPlusTreeError { impl<'a> SlottedPage<'a> { - pub fn new(data: &'a mut [u8], page_type: PageType) -> Result { if data.len() < PAGE_HEADER_SIZE_USIZE { return Err(PageError::NoSpace); @@ -366,7 +364,7 @@ impl<'a> SlottedPage<'a> { let slot_pos = PAGE_HEADER_SIZE_USIZE + (index * SLOT_SIZE); // Safe slice access if slot_pos + 2 > self.data.len() { return None; } - let offset = u16::from_le_bytes(self.data[slot_pos..slot_pos+2].try_into().ok()?); + let offset = u16::from_le_bytes(self.data[slot_pos..slot_pos + 2].try_into().ok()?); // Bounds check for length header if (offset as usize) + 4 > self.data.len() { return None; } @@ -379,7 +377,7 @@ impl<'a> SlottedPage<'a> { pub fn get_cell_offset(&self, index: usize) -> Option { let slot_pos = PAGE_HEADER_SIZE_USIZE + (index * SLOT_SIZE); if slot_pos + 2 > self.data.len() { return None; } - Some(u16::from_le_bytes(self.data[slot_pos..slot_pos+2].try_into().ok()?)) + Some(u16::from_le_bytes(self.data[slot_pos..slot_pos + 2].try_into().ok()?)) } pub fn compact(&mut self) -> Result<(), PageError> { @@ -433,7 +431,7 @@ impl<'a> SlottedPage<'a> { return Err(PageError::InvalidIndex); // Cannot split empty page } if count == 1 { - // Cannot split single item fundamentally. + // Cannot split single item fundamentally. // Return Ok(None) explicitly to indicate no-op. return Ok(None); } @@ -524,7 +522,7 @@ struct ValueInfo { mode: ValueStorageMode, length: u32, #[serde(skip)] - compressed_cache: Option<(u8, Vec)> // (flag, payload) + compressed_cache: Option<(u8, Vec)>, // (flag, payload) } @@ -664,6 +662,14 @@ where } } + pub fn len(&self) -> usize { + if self.is_leaf { + self.keys.len() + } else { + self.children.iter().map(BPlusTreeNode::len).sum() + } + } + pub fn traverse(&self, visit: &mut F) where F: FnMut(&Vec, &Vec), @@ -1388,6 +1394,14 @@ where } } + pub fn is_empty(&self) -> bool { + self.root.keys.is_empty() + } + + pub fn len(&self) -> usize { + self.root.len() + } + pub fn insert(&mut self, key: K, value: V) { self.dirty = true; if self.root.keys.is_empty() { @@ -1459,6 +1473,97 @@ where } } + /// Bulk build a tree from pre-calculated `ValueInfos` (streaming compact helper). + /// Writes nodes to `file` starting at `start_offset`. + /// Returns the offset of the root node used to update the file header. + fn bulk_build_from_infos( + &mut self, + file: &mut W, + items: &[(K, ValueInfo)], + start_offset: u64, + ) -> io::Result { + if items.is_empty() { + // Write empty root + let root = BPlusTreeNode::::new(true); + let mut buffer = Vec::with_capacity(PAGE_SIZE_USIZE); + let _offset = root.serialize_to_block(file, &mut buffer, start_offset)?; + // For a single empty root, the root offset is the start offset. + // serialize_to_block returns the *end* offset (start of next block). + // But header needs the *start* of the root node. + // Wait - serialize_to_block logic: + // "Returns the offset of the NEXT block" + // So the root IS at start_offset. + return Ok(start_offset); + } + + // 1. Sort items (Query iterator is usually sorted, but let's be safe or rely on caller) + // Since BPlusTreeQuery works on offsets, order might be physical not logical if we iterate blocks? + // BPlusTreeQuery::iter() uses range_scan which traverses logical order. So it IS sorted! + + let mut current_offset = start_offset; + let mut write_buffer = Vec::with_capacity(PAGE_SIZE_USIZE); + + // 2. Build Leaves + // Chunk items into leaf nodes + let mut next_level_pointers: Vec<(K, u64)> = Vec::new(); + + for chunk in items.chunks(self.leaf_order) { + let mut node = BPlusTreeNode::::new(true); + for (k, v) in chunk { + node.keys.push(k.clone()); + node.value_info.push(v.clone()); + } + + let node_offset = current_offset; + current_offset = node.serialize_to_block(file, &mut write_buffer, node_offset)?; + + if !node.keys.is_empty() { + next_level_pointers.push((node.keys[0].clone(), node_offset)); + } + } + + // 3. Build Internal Levels recursively + while next_level_pointers.len() > 1 { + let mut parent_level_pointers: Vec<(K, u64)> = Vec::new(); + let children = next_level_pointers; // Move + + // Chunk size: `inner_order + 1` (pointers). + for chunk in children.chunks(self.inner_order + 1) { + let mut node = BPlusTreeNode::::new(false); + let mut pointers = Vec::new(); + + // First pointer + if let Some((_, off)) = chunk.first() { + pointers.push(*off); + } + + // Subsequent pointers and separators + for (k, off) in &chunk[1..] { + node.keys.push(k.clone()); + pointers.push(*off); + } + + let node_offset = current_offset; + + // Use serialize_internal_with_offsets + current_offset = node.serialize_internal_with_offsets(file, &mut write_buffer, node_offset, &pointers)?; + + // Track for upper level + if let Some((k, _)) = chunk.first() { + parent_level_pointers.push((k.clone(), node_offset)); + } + } + next_level_pointers = parent_level_pointers; + } + + // 4. Return Root Offset + if let Some((_, root_off)) = next_level_pointers.first() { + Ok(*root_off) + } else { + Ok(start_offset) + } + } + pub fn load(filepath: &Path) -> io::Result { let mut file = File::open(filepath)?; @@ -1643,6 +1748,50 @@ where } } +fn count_items( + file: &mut R, + buffer: &mut Vec, + cache: &mut IndexMap>, + start_offset: u64, +) -> Result +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut count = 0; + let mut stack = vec![start_offset]; + while let Some(offset) = stack.pop() { + let (node, pointers) = if let Some(cached_block) = cache.get_mut(&offset) { + let data = cached_block.clone(); + let temp_buffer = data; + let res = BPlusTreeNode::::deserialize_from_block_slice(&temp_buffer, file, false) + .map_err(BPlusTreeError::from)?; + if let Some(removed) = cache.shift_remove(&offset) { + cache.insert(offset, removed); + } + res + } else { + match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { + Ok((node, pointers)) => { + if cache.len() >= CACHE_CAPACITY { + cache.shift_remove_index(0); + } + cache.insert(offset, buffer.to_owned()); + (node, pointers) + } + Err(err) => return Err(BPlusTreeError::from(err)), + } + }; + + if node.is_leaf { + count += node.keys.len(); + } else if let Some(ptrs) = pointers { + stack.extend(ptrs); + } + } + Ok(count) +} + /// `BPlusTreeQuery` performs on-disk queries without loading the entire tree into memory. /// For frequent queries, consider using `BPlusTree::load()` instead, which loads the full tree into memory @@ -1695,6 +1844,20 @@ where query_tree(&mut self.file, &mut self.buffer, &mut self.cache, key, self.root_offset) } + pub fn is_empty(&mut self) -> Result { + let (node, _) = BPlusTreeNode::::deserialize_from_block( + &mut self.file, + &mut self.buffer, + self.root_offset, + false, + )?; + Ok(node.is_leaf && node.keys.is_empty()) + } + + pub fn len(&mut self) -> Result { + count_items::(&mut self.file, &mut self.buffer, &mut self.cache, self.root_offset) + } + pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { query_tree_le(&mut self.file, &mut self.buffer, &mut self.cache, key, self.root_offset) } @@ -2002,97 +2165,94 @@ where query_tree(&mut reader, &mut self.read_buffer, &mut self.cache, key, self.root_offset) } + pub fn is_empty(&mut self) -> Result { + let mut reader = utils::file_reader(&mut self.file); + let (node, _) = BPlusTreeNode::::deserialize_from_block( + &mut reader, + &mut self.read_buffer, + self.root_offset, + false, + )?; + Ok(node.is_leaf && node.keys.is_empty()) + } + + pub fn len(&mut self) -> Result { + let mut reader = utils::file_reader(&mut self.file); + count_items::(&mut reader, &mut self.read_buffer, &mut self.cache, self.root_offset) + } + pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { let mut reader = utils::file_reader(&mut self.file); query_tree_le(&mut reader, &mut self.read_buffer, &mut self.cache, key, self.root_offset) } - fn update_recursive( - &mut self, - offset: u64, - key: &K, - value: &V, - ) -> Result { - let mut reader = utils::file_reader(&mut self.file); - let (mut node, pointers) = BPlusTreeNode::::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false)?; - - if node.is_leaf { - match node.keys.binary_search(key) { - Ok(idx) => { - // COW: Write new value with adaptive LZ4 compression - let raw_bytes = binary_serialize(value)?; - - - - let (flag, payload) = compress_if_beneficial(&raw_bytes); - - self.file.seek(SeekFrom::End(0))?; - let val_offset = self.file.stream_position()?; - - // Write: [flag:1][payload] - self.file.write_all(&[flag])?; - // NOTE: For LZ4, payload already includes original length (prepended) - self.file.write_all(&payload)?; - - // stored_len includes flag + payload - let stored_len = 1 + payload.len(); - - // Update leaf metadata - node.value_info[idx] = ValueInfo { - mode: ValueStorageMode::Single(val_offset), - length: u32::try_from(stored_len).map_err(to_io_error)?, - compressed_cache: None, - }; - - // Write new leaf node at end of file - self.file.seek(SeekFrom::End(0))?; - let new_leaf_offset = self.file.stream_position()?; - node.serialize_to_block(&mut self.file, &mut self.write_buffer, new_leaf_offset)?; - Ok(new_leaf_offset) - } - Err(_) => Err(BPlusTreeError::KeyNotFound), - } - } else { - let child_idx = get_entry_index_upper_bound::(&node.keys, key); - if let Some(mut pters) = pointers { - if let Some(&child_offset) = pters.get(child_idx) { - // Recurse to get a new child offset - let new_child_offset = self.update_recursive(child_offset, key, value)?; - - // Update current node's pointers - pters[child_idx] = new_child_offset; - - // COW: Write new internal node at end of file - self.file.seek(SeekFrom::End(0))?; - let new_node_offset = self.file.stream_position()?; - - // Use the robust helper to serialize the internal node with updated child pointers - node.serialize_internal_with_offsets(&mut self.file, &mut self.write_buffer, new_node_offset, &pters)?; - Ok(new_node_offset) - } else { - Err(BPlusTreeError::InvalidStructure("Child pointer not found in internal node".into())) - } - } else { - Err(BPlusTreeError::InvalidStructure("Internal node missing pointers invariant violation".into())) - } - } - } - - pub fn update(&mut self, key: &K, value: V) -> Result { - let new_root_offset = self.update_recursive(self.root_offset, key, &value)?; + let refs = [(key, &value)]; + let new_root_offset = self.update_batch_recursive(self.root_offset, &refs)?; // Atomic Header Swap - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; - self.file.write_all(&new_root_offset.to_le_bytes())?; - self.file.flush()?; - self.file.sync_all()?; + self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; + self.file.write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; + self.file.flush().map_err(BPlusTreeError::Io)?; + self.file.sync_all().map_err(BPlusTreeError::Io)?; self.root_offset = new_root_offset; Ok(new_root_offset) } + /// Update multiple items in batch. This is more efficient than calling `update()` multiple times + /// as it performs all updates and then commits the final root offset once. + /// returns The final root offset after all updates, or an error if any update fails + fn update_batch_recursive( + &mut self, + offset: u64, + items: &[(&K, &V)], + ) -> Result { + let mut reader = utils::file_reader(&mut self.file); + let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false)?; + + if node.is_leaf { + for (key, value) in items { + match node.keys.binary_search(key) { + Ok(idx) => { + let (val_off, val_len) = self.insert_value_to_disk(value).map_err(BPlusTreeError::Io)?; + node.value_info[idx] = ValueInfo { + mode: ValueStorageMode::Single(val_off), + length: val_len, + compressed_cache: None, + }; + } + Err(_) => return Err(BPlusTreeError::KeyNotFound), + } + } + let new_offset = self.write_node(&node).map_err(BPlusTreeError::Io)?; + Ok(new_offset) + } else { + let mut pointers = pointers_opt.ok_or_else(|| BPlusTreeError::InvalidStructure("Internal node missing pointers".into()))?; + + let mut current_idx = 0; + while current_idx < items.len() { + let first_key_in_group = items[current_idx].0; + let child_idx = get_entry_index_upper_bound::(&node.keys, first_key_in_group); + + let mut group_end = current_idx + 1; + while group_end < items.len() && get_entry_index_upper_bound::(&node.keys, items[group_end].0) == child_idx { + group_end += 1; + } + + let sub_items = &items[current_idx..group_end]; + let new_child_offset = self.update_batch_recursive(pointers[child_idx], sub_items)?; + pointers[child_idx] = new_child_offset; + + current_idx = group_end; + } + + let new_offset = self.write_internal_node(&node, &pointers).map_err(BPlusTreeError::Io)?; + Ok(new_offset) + } + } + /// Update multiple items in batch. This is more efficient than calling `update()` multiple times /// as it performs all updates and then commits the final root offset once. /// returns The final root offset after all updates, or an error if any update fails @@ -2101,21 +2261,19 @@ where return Ok(self.root_offset); } - let mut current_root = self.root_offset; + let mut sorted_items = items.to_vec(); + sorted_items.sort_by(|a, b| a.0.cmp(b.0)); - // Perform all updates sequentially - for (key, value) in items { - current_root = self.update_recursive(current_root, key, value)?; - } + let new_root_offset = self.update_batch_recursive(self.root_offset, &sorted_items)?; // Atomic Header Swap - only once at the end - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; - self.file.write_all(¤t_root.to_le_bytes())?; - self.file.flush()?; - self.file.sync_all()?; + self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; + self.file.write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; + self.file.flush().map_err(BPlusTreeError::Io)?; + self.file.sync_all().map_err(BPlusTreeError::Io)?; - self.root_offset = current_root; - Ok(current_root) + self.root_offset = new_root_offset; + Ok(new_root_offset) } @@ -2156,104 +2314,139 @@ where Ok(offset) } - fn upsert_recursive(&mut self, offset: u64, key: K, value: V) -> io::Result<(u64, Option<(K, u64)>)> { + + fn upsert_batch_recursive( + &mut self, + offset: u64, + items: &[(&K, &V)], + ) -> io::Result<(u64, Vec<(K, u64)>)> { + let mut reader = utils::file_reader(&mut self.file); let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block( - &mut self.file, + &mut reader, &mut self.read_buffer, offset, false, // shallow )?; if node.is_leaf { - match node.keys.binary_search(&key) { - Ok(idx) => { - let (val_off, val_len) = self.insert_value_to_disk(&value)?; - let new_info = ValueInfo { mode: ValueStorageMode::Single(val_off), length: val_len, compressed_cache: None }; - node.value_info[idx] = new_info; + for (key, value) in items { + let (val_off, val_len) = self.insert_value_to_disk(value)?; + let new_info = ValueInfo { mode: ValueStorageMode::Single(val_off), length: val_len, compressed_cache: None }; - let new_offset = self.write_node(&node)?; - Ok((new_offset, None)) - } - Err(idx) => { - let (val_off, val_len) = self.insert_value_to_disk(&value)?; - let new_info = ValueInfo { mode: ValueStorageMode::Single(val_off), length: val_len, compressed_cache: None }; - node.keys.insert(idx, key); - node.value_info.insert(idx, new_info); - - if node.keys.len() > self.leaf_order { - let median = self.leaf_order >> 1; - let mut right_node = BPlusTreeNode::new(true); - right_node.keys = node.keys.split_off(median); - right_node.value_info = node.value_info.split_off(median); - - let promoted_key = right_node.keys.first().ok_or_else(|| io::Error::other("Split resulted in empty right node keys"))?.clone(); - - let left_offset = self.write_node(&node)?; - let right_offset = self.write_node(&right_node)?; - - Ok((left_offset, Some((promoted_key, right_offset)))) - } else { - let new_offset = self.write_node(&node)?; - Ok((new_offset, None)) + match node.keys.binary_search(key) { + Ok(idx) => { + node.value_info[idx] = new_info; + } + Err(idx) => { + node.keys.insert(idx, (*key).clone()); + node.value_info.insert(idx, new_info); } } } - } else { - let mut pointers = pointers_opt.unwrap(); - let idx = get_entry_index_upper_bound(&node.keys, &key); - let child_offset = *pointers.get(idx).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Pointer index out of bounds"))?; - let (new_child_offset, split_res) = self.upsert_recursive(child_offset, key, value)?; - pointers[idx] = new_child_offset; + let mut leaf_promotions = Vec::new(); + while node.keys.len() > self.leaf_order { + let median_idx = node.keys.len() / 2; + let mut right_node = BPlusTreeNode::new(true); + right_node.keys = node.keys.split_off(median_idx); + right_node.value_info = node.value_info.split_off(median_idx); - if let Some((median_key, right_child_offset)) = split_res { - node.keys.insert(idx, median_key); - pointers.insert(idx + 1, right_child_offset); - - if node.keys.len() > self.inner_order { - let median = self.inner_order >> 1; - let mut right_node = BPlusTreeNode::new(false); - let right_pointers = pointers.split_off(median + 1); - right_node.keys = node.keys.split_off(median + 1); - - let promoted_key = node.keys.pop().unwrap(); - - let left_offset = self.write_internal_node(&node, &pointers)?; - let right_offset = self.write_internal_node(&right_node, &right_pointers)?; - - Ok((left_offset, Some((promoted_key, right_offset)))) - } else { - let new_offset = self.write_internal_node(&node, &pointers)?; - Ok((new_offset, None)) - } - } else { - let new_offset = self.write_internal_node(&node, &pointers)?; - Ok((new_offset, None)) + let promoted_key = right_node.keys[0].clone(); + let right_offset = self.write_node(&right_node)?; + leaf_promotions.push((promoted_key, right_offset)); } + + let new_leaf_offset = self.write_node(&node)?; + Ok((new_leaf_offset, leaf_promotions)) + } else { + let mut pointers = pointers_opt.ok_or_else(|| io::Error::other("Internal node missing pointers"))?; + let mut current_idx = 0; + + while current_idx < items.len() { + let first_key_in_group = items[current_idx].0; + let child_idx = get_entry_index_upper_bound::(&node.keys, first_key_in_group); + + let mut group_end = current_idx + 1; + while group_end < items.len() && get_entry_index_upper_bound::(&node.keys, items[group_end].0) == child_idx { + group_end += 1; + } + + let sub_items = &items[current_idx..group_end]; + let (new_child_offset, child_promotions) = self.upsert_batch_recursive(pointers[child_idx], sub_items)?; + pointers[child_idx] = new_child_offset; + + for (median_key, right_child_offset) in child_promotions { + let insert_idx = get_entry_index_upper_bound::(&node.keys, &median_key); + node.keys.insert(insert_idx, median_key); + pointers.insert(insert_idx + 1, right_child_offset); + } + + current_idx = group_end; + } + + let mut node_promotions = Vec::new(); + while node.keys.len() > self.inner_order { + let median_idx = node.keys.len() / 2; + let mut right_node = BPlusTreeNode::new(false); + + let promoted_key = node.keys.remove(median_idx); + right_node.keys = node.keys.split_off(median_idx); + let right_pointers = pointers.split_off(median_idx + 1); + + let right_offset = self.write_internal_node(&right_node, &right_pointers)?; + node_promotions.push((promoted_key, right_offset)); + } + + let new_offset = self.write_internal_node(&node, &pointers)?; + Ok((new_offset, node_promotions)) } } + fn build_higher_levels(&mut self, base_offset: u64, mut promotions: Vec<(K, u64)>) -> io::Result { + if promotions.is_empty() { + return Ok(base_offset); + } + promotions.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut node = BPlusTreeNode::::new(false); + let mut pointers = vec![base_offset]; + for (key, ptr) in promotions { + node.keys.push(key); + pointers.push(ptr); + } + + if node.keys.len() <= self.inner_order { + return self.write_internal_node(&node, &pointers); + } + + let mut next_level_promotions = Vec::new(); + while node.keys.len() > self.inner_order { + let median_idx = node.keys.len() / 2; + let mut right_node = BPlusTreeNode::new(false); + let promoted_key = node.keys.remove(median_idx); + right_node.keys = node.keys.split_off(median_idx); + let right_pointers = pointers.split_off(median_idx + 1); + let right_offset = self.write_internal_node(&right_node, &right_pointers)?; + next_level_promotions.push((promoted_key, right_offset)); + } + let left_offset = self.write_internal_node(&node, &pointers)?; + self.build_higher_levels(left_offset, next_level_promotions) + } + /// Insert or update multiple items in batch (upsert). - /// Uses disk-based recursive insertion (COW) to avoid loading full tree. + /// Uses disk-based recursive traversal for efficiency, processing each node only once. pub fn upsert_batch(&mut self, items: &[(&K, &V)]) -> io::Result { if items.is_empty() { return Ok(self.root_offset); } - let mut current_root = self.root_offset; + let mut sorted_items = items.to_vec(); + sorted_items.sort_by(|a, b| a.0.cmp(b.0)); - for (key, value) in items { - let (new_root, split) = self.upsert_recursive(current_root, (*key).clone(), (*value).clone())?; - current_root = new_root; + let (mut current_root, promotions) = self.upsert_batch_recursive(self.root_offset, &sorted_items)?; - if let Some((median_key, right_child)) = split { - let mut new_root_node = BPlusTreeNode::::new(false); - new_root_node.keys.push(median_key); - let pointers = vec![current_root, right_child]; - - current_root = self.write_internal_node(&new_root_node, &pointers)?; - } - } + // Handle promotions (splits) using balanced approach + current_root = self.build_higher_levels(current_root, promotions)?; self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; self.file.write_all(¤t_root.to_le_bytes())?; @@ -2266,13 +2459,65 @@ where /// Garbage Collection: Compacts the file by rewriting only live blocks sequentially. pub fn compact(&mut self, filepath: &Path) -> io::Result<()> { - // 1. Reload the current tree fully from the live root - let mut tree = BPlusTree::::load(filepath)?; - // 2. Setting dirty=true forces store_internal() to rewrite the file sequentially. - // We use store_internal because we already hold the lock in self._lock. - tree.dirty = true; - let new_root_offset = tree.store_internal(filepath)?; - self.root_offset = new_root_offset; + let mut value_infos: Vec<(K, ValueInfo)> = Vec::new(); + let mut temp_file = NamedTempFile::new_in(filepath.parent().unwrap_or(Path::new(".")))?; + + // 1. Write Header placeholder + temp_file.seek(SeekFrom::Start(0))?; + temp_file.write_all(MAGIC)?; + temp_file.write_all(&STORAGE_VERSION.to_le_bytes())?; + temp_file.seek(SeekFrom::Start(HEADER_SIZE))?; // Skip to start of content + + let mut current_offset = HEADER_SIZE; + + // 2. Iterate source and write values immediately (Streaming) + { + let mut query = BPlusTreeQuery::::try_new(filepath).map_err(to_io_error)?; + let mut write_buffer = std::io::BufWriter::new(&mut temp_file); + + for (k, v) in query.iter() { + let value_bytes = binary_serialize(&v)?; + let len = value_bytes.len(); + + let offset = current_offset; + + // Write value header (flag + payload) + write_buffer.write_all(&[COMPRESSION_FLAG_NONE])?; + write_buffer.write_all(&value_bytes)?; + + current_offset += 1 + len as u64; + + let stored_len = u32::try_from(len + 1).map_err(to_io_error)?; + value_infos.push((k, ValueInfo { + mode: ValueStorageMode::Single(offset), + length: stored_len, + compressed_cache: None, + })); + } + write_buffer.flush()?; + } + + // 3. Build Tree Structure + current_offset = temp_file.seek(SeekFrom::End(0))?; + + let mut tree = BPlusTree::::new(); + let root_offset = tree.bulk_build_from_infos(&mut temp_file, &value_infos, current_offset)?; + + // 4. Update Header + temp_file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; + temp_file.write_all(&root_offset.to_le_bytes())?; + + temp_file.flush()?; + temp_file.as_file().sync_all()?; + + // 5. Atomic Replace + temp_file.persist(filepath).map_err(to_io_error)?; + + // 6. Refresh state + self.root_offset = root_offset; + self.file = utils::open_read_write_file(filepath)?; + self.cache.clear(); + Ok(()) } } @@ -2482,6 +2727,81 @@ mod tests { }); } + #[test] + fn test_upsert_batch() -> io::Result<()> { + let tempdir = tempfile::tempdir()?; + let filepath = tempdir.path().join("upsert_batch_test.bin"); + + // 1. Create initial tree + let mut tree = BPlusTree::::new(); + tree.insert(1, Record { id: 1, data: "original 1".to_string() }); + tree.insert(2, Record { id: 2, data: "original 2".to_string() }); + tree.store(&filepath)?; + + // 2. Open for update and upsert batch + let mut update = BPlusTreeUpdate::::try_new(&filepath)?; + let r1_new = Record { id: 1, data: "updated 1".to_string() }; + let r3_new = Record { id: 3, data: "new 3".to_string() }; + + update.upsert_batch(&[ + (&1, &r1_new), + (&3, &r3_new), + ])?; + + // 3. Verify with query + let mut query = BPlusTreeQuery::::try_new(&filepath)?; + assert_eq!(query.query(&1).unwrap(), Some(r1_new)); + assert_eq!(query.query(&2).unwrap(), Some(Record { id: 2, data: "original 2".to_string() })); + assert_eq!(query.query(&3).unwrap(), Some(r3_new)); + + Ok(()) + } + + #[test] + fn len_test() -> io::Result<()> { + let test_size = 100; + let mut tree = BPlusTree::::new(); + + // Initial state + assert_eq!(tree.len(), 0); + assert!(tree.is_empty()); + + for i in 1..=test_size { + tree.insert(i, Record { + id: i, + data: format!("data {i}"), + }); + assert_eq!(tree.len(), i as usize); + assert!(!tree.is_empty()); + } + + let tempdir = tempfile::tempdir()?; + let filepath = tempdir.path().join("len_test.bin"); + tree.store(&filepath)?; + + // Test BPlusTreeQuery len + let mut query: BPlusTreeQuery = BPlusTreeQuery::try_new(&filepath)?; + assert_eq!(query.len().expect("Query len failed"), test_size as usize); + assert!(!query.is_empty().expect("Query is_empty failed")); + + // Test BPlusTreeUpdate len and modifications + let mut update: BPlusTreeUpdate = BPlusTreeUpdate::try_new(&filepath)?; + assert_eq!(update.len().expect("Update len failed"), test_size as usize); + + // Update existing key - length should stay same + update.update(&1, Record { id: 1, data: "updated".to_string() }).map_err(|e| e.to_io())?; + assert_eq!(update.len().expect("Update len failed after update"), test_size as usize); + + // Insert new key - length should increase + update.upsert_batch(&[ + (&(test_size + 1), &Record { id: test_size + 1, data: "new".to_string() }) + ])?; + assert_eq!(update.len().expect("Update len failed after insert"), (test_size + 1) as usize); + + Ok(()) + } + + #[test] fn iterator_test() -> io::Result<()> { let mut tree = BPlusTree::::new(); @@ -2922,6 +3242,43 @@ mod tests { Ok(()) } + + #[test] + fn compact_reopen_test() -> io::Result<()> { + let tempdir = tempfile::tempdir()?; + let filepath = tempdir.path().join("tree_compact_reopen.bin"); + let mut tree = BPlusTree::::new(); + + // Initial write + tree.insert(1, Record { id: 1, data: "Initial".to_string() }); + tree.store(&filepath)?; + drop(tree); + + let mut tree_update = BPlusTreeUpdate::::try_new(&filepath)?; + + // 1. Write something + let r2 = Record { id: 2, data: "BeforeCompact".to_string() }; + tree_update.upsert_batch(&[(&2, &r2)])?; + + // 2. Compact (this replaces the file) + tree_update.compact(&filepath)?; + + // 3. Write something else + let r3 = Record { id: 3, data: "AfterCompact".to_string() }; + tree_update.upsert_batch(&[(&3, &r3)])?; + + drop(tree_update); + + // Verify all data is present in the NEW file + let mut tree_check = BPlusTreeQuery::::try_new(&filepath)?; + + assert!(tree_check.query(&1).map_err(|e| e.to_io())?.is_some(), "Should have key 1"); + assert!(tree_check.query(&2).map_err(|e| e.to_io())?.is_some(), "Should have key 2"); + assert!(tree_check.query(&3).map_err(|e| e.to_io())?.is_some(), "Should have key 3 - if missing, file handle wasn't updated"); + + Ok(()) + } + #[test] fn upsert_batch_mixed_test() -> io::Result<()> { let tempdir = tempfile::tempdir()?; @@ -3091,7 +3448,7 @@ mod tests { let file_size = std::fs::metadata(&filepath)?.len(); - // Expected size without packing: + // Expected size without packing: // 1000 items * 4096 bytes/block = 4,096,000 bytes (~4MB) // Plus internal nodes let unpacked_size_estimate = count as u64 * super::PAGE_SIZE_USIZE as u64; @@ -3099,9 +3456,9 @@ mod tests { println!("File size with packing: {} bytes", file_size); println!("Estimated unpacked size: {} bytes", unpacked_size_estimate); - // We expect significant savings. + // We expect significant savings. // 1000 items * ~60 bytes / 4096 bytes/block ~= 15 blocks - // Plus tree structure overhead. + // Plus tree structure overhead. // Let's be conservative and say it should be less than 10% of unpacked size. assert!(file_size < unpacked_size_estimate / 10, "Packing should reduce size by at least 90%"); @@ -3193,7 +3550,7 @@ mod tests { let mut tree_update = BPlusTreeUpdate::::try_new(&filepath)?; - // Insert 5000 items. + // Insert 5000 items. // 5000 items ensures at least Root -> Internal -> Leaf split (Height 2 or 3). let count = 5000; @@ -3268,10 +3625,14 @@ mod tests { for i in 0..count { updates.push((i, val.clone())); } - let refs: Vec<(&u32, &String)> = updates.iter().map(|(k, v)| (k, v)).collect(); - tree_update.upsert_batch(&refs)?; + // Use individual upsert calls to create fragmentation (one path write per update) + for (k, v) in updates { + let refs = [(&k, &v)]; + tree_update.upsert_batch(&refs)?; + } let size_before = std::fs::metadata(&filepath)?.len(); + // Each individual update writes a full path, increasing file size significantly. assert!(size_before > count as u64 * 4000); // Now Compact @@ -3475,9 +3836,10 @@ mod page_tests { match res { Ok(None) => { assert_eq!(page.header.cell_count, 1); // Original page untouched - }, + } Ok(Some(_)) => panic!("Split of single item should result in None"), Err(e) => panic!("Split of single item should result in no-op, not error: {:?}", e), } } } + diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index 2a0c85c33..4a7bc91e1 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -19,14 +19,14 @@ const XML_PREAMBLE: &str = r#" // // // XML Header via events (DO NOT USE, kept for documentation): // writer.write_event_async(quick_xml::events::Event::Decl(quick_xml::events::BytesDecl::new("1.0", Some("utf-8"), None))) -// .await.map_err(|e| notify_err!(format!("failed to write XML header: {}", e)))?; +// .await.map_err(|e| notify_err!("failed to write XML header: {}", e))?; // // // DOCTYPE via events (DO NOT USE): // writer.write_event_async(quick_xml::events::Event::DocType(quick_xml::events::BytesText::new(r#"tv SYSTEM "xmltv.dtd""#))) -// .await.map_err(|e| notify_err!(format!("failed to write doctype: {}", e)))?; +// .await.map_err(|e| notify_err!("failed to write doctype: {}", e))?; pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playlist: Option<&[PlaylistGroup]>) -> Result<(), TuliproxError> { let file = tokio::fs::File::create(path).await - .map_err(|e| notify_err!(format!("failed to create epg file {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to create epg file {}: {}", path.display(), e))?; // Use a larger buffer for sequential writes to reduce syscalls let mut buf_writer = async_file_writer(file); @@ -35,7 +35,7 @@ pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playl buf_writer .write_all(XML_PREAMBLE.as_bytes()) .await - .map_err(|e| notify_err!(format!("failed to write XML preamble {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to write XML preamble {}: {}", path.display(), e))?; // Build a temporary rename map with zero allocations (uses references) let mut rename_map: HashMap<&str, &str> = HashMap::new(); @@ -56,16 +56,16 @@ pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playl epg .write_to_async(&mut writer, if rename_map.is_empty() { None } else { Some(&rename_map) }) .await - .map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to write epg {}: {}", path.display(), e))?; // Ensure buffers are flushed to the OS and capture any I/O error let mut buf_writer = writer.into_inner(); buf_writer .flush() .await - .map_err(|e| notify_err!(format!("failed to flush epg {}: {}", path.display(), e)))?; + .map_err(|e| notify_err!("failed to flush epg {}: {}", path.display(), e))?; - buf_writer.shutdown().await.map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?; + buf_writer.shutdown().await.map_err(|e| notify_err!("failed to write epg {}: {}", path.display(), e))?; debug_if_enabled!("Epg for target {} written to {}", target.name, path.display()); Ok(()) @@ -81,7 +81,7 @@ pub async fn epg_write(cfg: &Config, target: &ConfigTarget, target_path: &Path, debug_if_enabled!("writing xtream epg to {}", epg_path.display()); epg_write_file(target, epg_data, &epg_path, playlist).await?; } - None => return Err(notify_err!(format!("failed to serialize epg for target: {}, storage path not found", target.name))), + None => return Err(notify_err!("failed to write epg for target: {}, storage path not found", target.name)), } } TargetOutput::M3u(_) => { diff --git a/backend/src/repository/library_repository.rs b/backend/src/repository/library_repository.rs new file mode 100644 index 000000000..71f233213 --- /dev/null +++ b/backend/src/repository/library_repository.rs @@ -0,0 +1,54 @@ +use crate::model::AppConfig; +use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; +use shared::error::{TuliproxError, notify_err_res}; +use shared::model::{PlaylistGroup, PlaylistItem, UUIDType, XtreamPlaylistItem}; +use std::path::Path; +use std::sync::Arc; +use indexmap::IndexMap; + +pub async fn persist_input_library_playlist(app_config: &Arc, library_path: &Path, playlist: &[PlaylistGroup]) -> Result<(), TuliproxError> { + if playlist.is_empty() { + return Ok(()); + } + let _file_lock = app_config.file_locks.write_lock(library_path).await; + let mut tree = BPlusTree::new(); + for pg in playlist { + for item in &pg.channels { + let xtream = XtreamPlaylistItem::from(item); + tree.insert(item.header.uuid, xtream); + } + } + match tree.store(library_path) { + Ok(_) => Ok(()), + Err(err) => notify_err_res!("failed to write local library playlist: {} - {err}", library_path.display()) + } +} + + +pub async fn load_input_local_library_playlist(app_config: &Arc, lib_path: &Path) -> Result, TuliproxError> { + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); + + if tokio::fs::try_exists(lib_path).await.unwrap_or(false) { + // Load Items + let _file_lock = app_config.file_locks.read_lock(lib_path).await; + if let Ok(mut query) = BPlusTreeQuery::::try_new(lib_path) { + let mut group_cnt = 0; + for (_, ref item) in query.iter() { + let cat_id = item.group.clone(); + groups.entry(cat_id) + .or_insert_with(|| { + group_cnt += 1; + PlaylistGroup { + id: group_cnt, + title: item.group.clone(), + channels: Vec::new(), + xtream_cluster: item.xtream_cluster, + } + }) + .channels.push(PlaylistItem::from(item)); + } + } + } + + Ok(groups.into_values().collect()) +} diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index 6d63027c5..d47865856 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -37,7 +37,7 @@ impl M3uPlaylistIterator { // TODO use playlist memory cache, but be aware of sorting ! - let m3u_output = target.get_m3u_output().ok_or_else(|| info_err!(format!("Unexpected failure, missing m3u target output for target {}", target.name)))?; + let m3u_output = target.get_m3u_output().ok_or_else(|| info_err!("Unexpected failure, missing m3u target output for target {}", target.name))?; let config = cfg.config.load(); let target_path = ensure_target_storage_path(&config, target.name.as_str())?; let m3u_path = m3u_get_file_path(&target_path); @@ -45,7 +45,7 @@ impl M3uPlaylistIterator { let file_lock = cfg.file_locks.read_lock(&m3u_path).await; let query = BPlusTreeQuery::::try_new(&m3u_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {m3u_path:?} - {err}")))?; + .map_err(|err| info_err!("Could not open BPlusTreeQuery {m3u_path:?} - {err}"))?; let reader = query.disk_iter(); let filter = user_get_bouquet_filter(&config, &user.username, None, TargetType::M3u, XtreamCluster::Live).await; @@ -116,14 +116,14 @@ impl M3uPlaylistIterator { let entry = if let Some(set) = &self.filter { if let Some((current_item, _)) = self.lookup_item.take() { // Avoid cloning strings while filtering - let next_valid = self.reader.find(|(_, pli)| set.contains(pli.group.as_str())); + let next_valid = self.reader.find(|(_, pli)| set.contains(&*pli.group)); self.lookup_item = next_valid.map(|(_k, v)| (v, true)); // has_next handled by iterator usually, but here we just need the item let has_next = self.lookup_item.is_some(); Some((current_item, has_next)) } else { - let current_item = self.reader.find(|(_, item)| set.contains(item.group.as_str())); + let current_item = self.reader.find(|(_, item)| set.contains(&*item.group)); if let Some((_, item)) = current_item { - self.lookup_item = self.reader.find(|(_, item)| set.contains(item.group.as_str())).map(|(_, item)| (item, true)); + self.lookup_item = self.reader.find(|(_, item)| set.contains(&*item.group)).map(|(_, item)| (item, true)); let has_next = self.lookup_item.is_some(); Some((item, has_next)) } else { diff --git a/backend/src/repository/m3u_repository.rs b/backend/src/repository/m3u_repository.rs index 06b244153..d3fc925d9 100644 --- a/backend/src/repository/m3u_repository.rs +++ b/backend/src/repository/m3u_repository.rs @@ -9,8 +9,7 @@ use crate::utils; use crate::utils::{async_file_writer, IO_BUFFER_SIZE}; use indexmap::IndexMap; use log::error; -use shared::error::{create_tuliprox_error, info_err, string_to_io_error}; -use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind}; +use shared::error::{notify_err, info_err, string_to_io_error, str_to_io_error, TuliproxError}; use shared::model::{M3uPlaylistItem, PlaylistGroup}; use shared::model::{PlaylistItem, PlaylistItemType, XtreamCluster}; use std::io::Error; @@ -22,7 +21,7 @@ use tokio::task; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.display() ,$err) + notify_err!("failed to write m3u playlist: {} - {}", $path.display() ,$err) } } @@ -38,7 +37,7 @@ pub fn m3u_get_epg_file_path(target_path: &Path) -> PathBuf { macro_rules! await_playlist_write { ($expr:expr, $fmt:literal $(, $args:expr)* ) => {{ $expr.await.map_err(|err| { - create_tuliprox_error!(TuliproxErrorKind::Notify, $fmt $(, $args)*, err) + notify_err!($fmt $(, $args)*, err) })? }}; } @@ -116,7 +115,7 @@ pub async fn m3u_write_playlist( Ok(()) }) .await - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {err}", m3u_path.display()))??; + .map_err(|err| notify_err!("failed to write m3u playlist: {} - {err}", m3u_path.display()))??; Ok(()) } @@ -165,7 +164,7 @@ pub async fn iter_raw_m3u_playlist(config: &AppConfig, target: &ConfigTarget) -> } let file_lock = config.file_locks.read_lock(&m3u_path).await; match BPlusTreeQuery::::try_new(&m3u_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {m3u_path:?} - {err}"))) { + .map_err(|err| info_err!("Could not open BPlusTreeQuery {m3u_path:?} - {err}")) { Ok(mut query) => { let items: Vec = query.iter().map(|(_, v)| v).collect(); let len = items.len(); @@ -196,13 +195,13 @@ pub async fn persist_input_m3u_playlist(app_config: &Arc, m3u_path: & Ok(()) }) .await - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {err}", m3u_path.display()))??; + .map_err(|err| notify_err!("failed to write m3u playlist: {} - {err}", m3u_path.display()))??; Ok(()) } pub async fn load_input_m3u_playlist(app_config: &Arc, m3u_path: &Path) -> Result, TuliproxError> { - let mut groups: IndexMap = IndexMap::new(); + let mut groups: IndexMap, PlaylistGroup> = IndexMap::new(); if tokio::fs::try_exists(m3u_path).await.unwrap_or(false) { // Load Items diff --git a/backend/src/repository/mod.rs b/backend/src/repository/mod.rs index 72ca6ef59..b1f80a820 100644 --- a/backend/src/repository/mod.rs +++ b/backend/src/repository/mod.rs @@ -11,4 +11,8 @@ pub mod xtream_playlist_iterator; pub mod user_repository; pub mod storage_const; mod playlist_scratch; +mod playlist_source; +mod library_repository; + +pub use playlist_source::*; diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 7fe7ad4d1..8a7ae5f33 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -1,7 +1,7 @@ use crate::api::model::{AppState, PlaylistM3uStorage, PlaylistStorage, PlaylistStorageState, PlaylistXtreamStorage}; use crate::model::Epg; use crate::model::{AppConfig, ConfigInput, ConfigTarget, TargetOutput}; -use crate::processing::processor::playlist::apply_filter_to_playlist; +use crate::processing::processor::playlist::{apply_filter_to_playlist, PlaylistProcessingContext}; use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; use crate::repository::epg_repository::epg_write; use crate::repository::m3u_repository::{load_input_m3u_playlist, m3u_get_file_path, m3u_write_playlist, persist_input_m3u_playlist}; @@ -12,15 +12,15 @@ use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord}; use crate::repository::xtream_repository::{load_input_xtream_playlist, persist_input_xtream_playlist, xtream_get_file_path, xtream_get_storage_path, xtream_write_playlist}; use crate::utils; use log::info; -use shared::create_tuliprox_error; -use shared::error::TuliproxError; -use shared::error::{info_err, TuliproxErrorKind}; +use crate::repository::{LocalLibraryDiskPlaylistSource, M3uDiskPlaylistSource, MemoryPlaylistSource, PlaylistSource, XtreamDiskPlaylistSource}; +use shared::error::{info_err, TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{InputType, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{is_dash_url, is_hls_url}; +use shared::utils::{is_dash_url, is_hls_url, StringInterner}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::repository::library_repository::{load_input_local_library_playlist, persist_input_library_playlist}; struct LocalEpisodeKey { path: String, @@ -33,7 +33,8 @@ pub struct ProviderEpisodeKey { } pub async fn persist_playlist(app_config: &Arc, playlist: &mut [PlaylistGroup], epg: Option<&Epg>, - target: &ConfigTarget, playlist_state: Option<&Arc>) -> Result<(), Vec> { + target: &ConfigTarget, playlist_state: Option<&Arc>, + interner: &mut StringInterner) -> Result<(), Vec> { let mut errors = vec![]; let config = &app_config.config.load(); let target_path = match ensure_target_storage_path(config, &target.name) { @@ -80,7 +81,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl rewrite_series_info_episode_virtual_id(playlist, &local_library_series, &provider_series); for output in &target.output { - let mut filtered = match output { + let mut filtered: Option> = match output { TargetOutput::Xtream(out) => out.filter.as_ref().and_then(|flt| apply_filter_to_playlist(playlist, flt)), TargetOutput::M3u(out) => out.filter.as_ref().and_then(|flt| apply_filter_to_playlist(playlist, flt)), TargetOutput::Strm(out) => out.filter.as_ref().and_then(|flt| apply_filter_to_playlist(playlist, flt)), @@ -94,7 +95,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl }; let result = match output { - TargetOutput::Xtream(_xtream_output) => xtream_write_playlist(app_config, target, pl).await, + TargetOutput::Xtream(_xtream_output) => xtream_write_playlist(app_config, target, pl, interner).await, TargetOutput::M3u(m3u_output) => m3u_write_playlist(app_config, target, m3u_output, &target_path, pl).await, TargetOutput::Strm(strm_output) => write_strm_playlist(app_config, target, strm_output, pl).await, TargetOutput::HdHomeRun(_hdhomerun_output) => Ok(()), @@ -114,7 +115,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl } if let Err(err) = target_id_mapping.persist() { - errors.push(info_err!(err.to_string())); + errors.push(info_err!("{err}")); } drop(file_lock); @@ -244,7 +245,7 @@ async fn load_target_id_mapping_as_tree(app_config: &AppConfig, target_path: &Pa let target_id_mapping_file = get_target_id_mapping_file(target_path); let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; BPlusTree::::load(&target_id_mapping_file).map_err(|err| - create_tuliprox_error!(TuliproxErrorKind::Info, "Could not find path for target {} err:{err}", &target.name)) + info_err!("Could not find path for target {} err:{err}", &target.name)) } async fn load_xtream_playlist_as_tree(app_config: &AppConfig, storage_path: &Path, cluster: XtreamCluster) -> BPlusTree { @@ -262,15 +263,10 @@ async fn load_xtream_playlist_as_tree(app_config: &AppConfig, storage_path: &Pat async fn load_xtream_target_storage(app_config: &AppConfig, target: &ConfigTarget) -> Result { let config = app_config.config.load(); let target_path = get_target_storage_path(&config, target.name.as_str()).ok_or_else(|| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {}", &target.name - ))?; + info_err!("Could not find path for target {}", &target.name))?; let storage_path = xtream_get_storage_path(&config, target.name.as_str()).ok_or_else(|| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {} xtream output", &target.name))?; + info_err!("Could not find path for target {} xtream output", &target.name))?; let target_id_mapping = load_target_id_mapping_as_tree(app_config, &target_path, target).await?; let live_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Live).await; @@ -288,10 +284,7 @@ async fn load_xtream_target_storage(app_config: &AppConfig, target: &ConfigTarge async fn load_m3u_target_storage(app_config: &AppConfig, target: &ConfigTarget) -> Result { let config = app_config.config.load(); let target_path = get_target_storage_path(&config, target.name.as_str()).ok_or_else(|| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {}", &target.name - ))?; + info_err!("Could not find path for target {}", &target.name))?; let m3u_path = m3u_get_file_path(&target_path); let _file_lock = app_config.file_locks.read_lock(&m3u_path).await; @@ -344,57 +337,99 @@ pub async fn persist_input_playlist(app_config: &Arc, input: &ConfigI let storage_path = match get_input_storage_path(&input.name, working_dir) { Ok(storage_path) => storage_path, Err(err) => { - return (playlist, Some(create_tuliprox_error!( TuliproxErrorKind::Info, "Error creating input storage directory for input '{}' failed: {err}", input.name))); + return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); } }; persist_input_xtream_playlist(app_config, &storage_path, playlist).await } - _ => { - // Persist M3U/Other types + InputType::M3u | InputType::M3uBatch => { + // Persist M3U let working_dir = &app_config.config.load().working_dir; let storage_path = match get_input_storage_path(&input.name, working_dir) { Ok(storage_path) => storage_path, Err(err) => { - return (playlist, Some(create_tuliprox_error!( TuliproxErrorKind::Info, "Error creating input storage directory for input '{}' failed: {err}", input.name))); + return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); } }; - let file_path = get_input_playlist_file_path(&storage_path, input.name.as_str()); + let file_path = get_input_m3u_playlist_file_path(&storage_path, input.name.as_str()); if let Err(err) = persist_input_m3u_playlist(app_config, &file_path, &playlist).await { return (playlist, Some(err)); } (playlist, None) + }, + InputType::Library => { + // Persist local library playlist + let working_dir = &app_config.config.load().working_dir; + let storage_path = match get_input_storage_path(&input.name, working_dir) { + Ok(storage_path) => storage_path, + Err(err) => { + return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); + } + }; + let file_path = get_input_local_library_playlist_file_path(&storage_path, input.name.as_str()); + if let Err(err) = persist_input_library_playlist(app_config, &file_path, &playlist).await { + return (playlist, Some(err)); + } + (playlist, None) } } } -pub async fn load_input_playlist(app_config: &Arc, input: &ConfigInput, clusters: Option<&[XtreamCluster]>) -> Result, TuliproxError> { +pub async fn load_input_playlist(ctx: &PlaylistProcessingContext, input: &ConfigInput, clusters: Option<&[XtreamCluster]>) -> Result, TuliproxError> { + let app_config = &ctx.config; let working_dir = &app_config.config.load().working_dir; let storage_path = get_input_storage_path(&input.name, working_dir) - .map_err(|e| create_tuliprox_error!(TuliproxErrorKind::Info, "Error getting input path: {e}"))?; + .map_err(|e| info_err!("Error getting input path: {e}"))?; + + let disk_based_processing = app_config.config.load().disk_based_processing; match input.input_type { InputType::Xtream | InputType::XtreamBatch => { - let clusters_to_load = if let Some(c) = clusters { - c + if disk_based_processing { + Ok(Box::new(XtreamDiskPlaylistSource::new(app_config, &storage_path).await)) } else { - &XTREAM_CLUSTER - }; - - load_input_xtream_playlist(app_config, &storage_path, clusters_to_load).await + let clusters_to_load = if let Some(c) = clusters { + c + } else { + &XTREAM_CLUSTER + }; + let groups = load_input_xtream_playlist(app_config, &storage_path, clusters_to_load).await?; + Ok(Box::new(MemoryPlaylistSource::new(groups))) + } } - _ => { + InputType::M3u | InputType::M3uBatch => { // Load M3U - let file_path = get_input_playlist_file_path(&storage_path, input.name.as_str()); - load_input_m3u_playlist(app_config, &file_path).await + let file_path = get_input_m3u_playlist_file_path(&storage_path, input.name.as_str()); + if disk_based_processing && file_path.exists() { + Ok(Box::new(M3uDiskPlaylistSource::new(app_config, &file_path).await)) + } else { + let groups = load_input_m3u_playlist(app_config, &file_path).await?; + Ok(Box::new(MemoryPlaylistSource::new(groups))) + } + } + InputType::Library => { + let file_path = get_input_local_library_playlist_file_path(&storage_path, input.name.as_str()); + if disk_based_processing && file_path.exists() { + Ok(Box::new(LocalLibraryDiskPlaylistSource::new(app_config, &file_path).await)) + } else { + let groups = load_input_local_library_playlist(app_config, &file_path).await?; + Ok(Box::new(MemoryPlaylistSource::new(groups))) + } } } } - -fn get_input_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { +fn get_input_m3u_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { let sanitized_input_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); - storage_path.join(format!("{sanitized_input_name}_playlist.{FILE_SUFFIX_DB}")) + storage_path.join(format!("m3u_{sanitized_input_name}.{FILE_SUFFIX_DB}")) +} + +fn get_input_local_library_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { + let sanitized_input_name: String = input_name.chars() + .map(|c| if c.is_alphanumeric() { c } else { '_' }) + .collect(); + storage_path.join(format!("lib_{sanitized_input_name}.{FILE_SUFFIX_DB}")) } \ No newline at end of file diff --git a/backend/src/repository/playlist_source.rs b/backend/src/repository/playlist_source.rs new file mode 100644 index 000000000..c16be6390 --- /dev/null +++ b/backend/src/repository/playlist_source.rs @@ -0,0 +1,525 @@ +use crate::model::AppConfig; +use crate::repository::bplustree::BPlusTreeQuery; +use crate::repository::xtream_repository::xtream_get_file_path; +use crate::utils::FileReadGuard; +use futures::future::BoxFuture; +use indexmap::IndexMap; +use log::{error, warn}; +use serde::{Deserialize, Serialize}; +use shared::model::{M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, UUIDType, XtreamCluster, XtreamPlaylistItem}; +use std::borrow::Cow; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub trait PlaylistSource: Send + Sync { + fn is_memory(&self) -> bool; + fn get_channel_count(&mut self) -> usize; + fn get_group_count(&mut self) -> usize; + fn is_empty(&mut self) -> bool; + #[allow(clippy::wrong_self_convention)] + fn into_items(&mut self) -> Box + Send + '_>; + fn items_mut(&mut self) -> Box + Send + '_>; + fn items<'a>(&'a mut self) -> Box> + Send + 'a>; + fn update_playlist<'a>(&'a mut self, plg: &'a PlaylistGroup) -> BoxFuture<'a, ()>; + fn get_missing_vod_info_count(&mut self) -> usize; + fn get_missing_series_info_count(&mut self) -> usize; + fn deduplicate(&mut self, duplicates: &mut HashSet); + fn take_groups(&mut self) -> Vec; + fn clone_box(&self) -> Box; + fn release_resources(&mut self, cluster: XtreamCluster); + fn obtain_resources(&mut self) -> BoxFuture<'_, ()>; + fn sort_by_provider_ordinal(&mut self); +} + +#[derive(Default)] +pub struct EmptyPlaylistSource {} + +impl PlaylistSource for EmptyPlaylistSource { + fn is_memory(&self) -> bool { true } + fn get_channel_count(&mut self) -> usize { 0 } + fn get_group_count(&mut self) -> usize { 0 } + fn is_empty(&mut self) -> bool { true } + fn into_items(&mut self) -> Box + Send + '_> { Box::new(std::iter::empty()) } + fn items_mut(&mut self) -> Box + Send + '_> { Box::new(std::iter::empty()) } + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { Box::new(std::iter::empty()) } + fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { Box::pin(async move {}) } + fn get_missing_vod_info_count(&mut self) -> usize { 0 } + fn get_missing_series_info_count(&mut self) -> usize { 0 } + fn deduplicate(&mut self, _duplicates: &mut HashSet) { /* noop */ } + fn take_groups(&mut self) -> Vec { vec![] } + fn clone_box(&self) -> Box { Box::new(EmptyPlaylistSource::default()) } + fn release_resources(&mut self, _cluster: XtreamCluster) { /* noop */ } + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { Box::pin(async move {}) } + fn sort_by_provider_ordinal(&mut self) { /* noop */ } +} + +pub struct XtreamDiskPlaylistSource { + app_config: Arc, + storage_path: PathBuf, + live: Option<(BPlusTreeQuery, Arc)>, + vod: Option<(BPlusTreeQuery, Arc)>, + series: Option<(BPlusTreeQuery, Arc)>, +} + +impl XtreamDiskPlaylistSource { + pub(crate) async fn new(app_config: &Arc, storage_path: &Path) -> Self { + let mut source = XtreamDiskPlaylistSource { + app_config: Arc::clone(app_config), + storage_path: storage_path.to_path_buf(), + live: None, + vod: None, + series: None, + }; + source.reload().await; + source + } + + async fn reload(&mut self) { + if self.live.is_none() { + let live_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Live); + self.live = load_bplustree_query::(&self.app_config, &live_path).await + .map(|(query, guard)| (query, Arc::new(guard))); + } + if self.vod.is_none() { + let vod_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Video); + self.vod = load_bplustree_query::(&self.app_config, &vod_path).await + .map(|(query, guard)| (query, Arc::new(guard))); + } + + if self.series.is_none() { + let series_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Series); + self.series = load_bplustree_query::(&self.app_config, &series_path).await + .map(|(query, guard)| (query, Arc::new(guard))); + } + } +} + +impl PlaylistSource for XtreamDiskPlaylistSource { + fn is_memory(&self) -> bool { false } + + fn get_channel_count(&mut self) -> usize { + self.live.as_mut().map_or(0usize, |(t, _)| t.len().unwrap_or(0usize)) + + self.vod.as_mut().map_or(0usize, |(t, _)| t.len().unwrap_or(0usize)) + + self.series.as_mut().map_or(0usize, |(t, _)| t.len().unwrap_or(0usize)) + } + + fn get_group_count(&mut self) -> usize { + let mut groups = HashSet::new(); + if let Some((query, _)) = self.live.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + if let Some((query, _)) = self.vod.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + if let Some((query, _)) = self.series.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + groups.len() + } + + fn is_empty(&mut self) -> bool { + self.live.as_mut().is_none_or(|(q, _)| q.is_empty().unwrap_or(true)) + && self.vod.as_mut().is_none_or(|(q, _)| q.is_empty().unwrap_or(true)) + && self.series.as_mut().is_none_or(|(q, _)| q.is_empty().unwrap_or(true)) + } + + fn into_items(&mut self) -> Box + Send + '_> { + let live = self.live.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| PlaylistItem::from(&item)); + let vod = self.vod.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| PlaylistItem::from(&item)); + let series = self.series.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| PlaylistItem::from(&item)); + Box::new(live.chain(vod).chain(series)) + } + + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + let live = self.live.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + let vod = self.vod.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + let series = self.series.as_mut().into_iter().flat_map(|(q, _)| q.iter()).map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + Box::new(live.chain(vod).chain(series)) + } + + fn items_mut(&mut self) -> Box + Send + '_> { + warn!("Disk-based playlist sources are read-only. Use clone_source() and convert to memory for mutable access."); + Box::new(std::iter::empty()) + } + + fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { + Box::pin(async move { + warn!("update_playlist should not be called for Xtream Disk playlist"); + // // Drop read guards before write lock + // self.live = None; + // self.vod = None; + // self.series = None; + // + // let xtream_path = xtream_get_file_path(&self.storage_path, plg.xtream_cluster); + // { + // let _lock = self.app_config.file_locks.write_lock(&xtream_path).await; + // if let Ok(mut tree) = BPlusTreeUpdate::::try_new(&xtream_path) { + // let xtream_items: Vec = plg.channels.iter().map(XtreamPlaylistItem::from).collect(); + // let batch: Vec<(&u32, &XtreamPlaylistItem)> = xtream_items.iter().map(|item| (&item.virtual_id, item)).collect(); + // let _ = tree.upsert_batch(&batch); + // } + // } + // self.reload().await; + }) + } + + fn get_missing_vod_info_count(&mut self) -> usize { + let mut count = 0; + if let Some((query, _)) = self.vod.as_mut() { + for (_, item) in query.iter() { + if item.item_type == PlaylistItemType::Video && item.provider_id > 0 && !item.has_details() { + count += 1; + } + } + } + count + } + + fn get_missing_series_info_count(&mut self) -> usize { + let mut count = 0; + if let Some((query, _)) = self.series.as_mut() { + for (_, item) in query.iter() { + if item.item_type == PlaylistItemType::SeriesInfo && item.provider_id > 0 && !item.has_details() { + count += 1; + } + } + } + count + } + + fn deduplicate(&mut self, _duplicates: &mut HashSet) { + warn!("Deduplication is not supported for disk based playlist updates"); + } + + fn take_groups(&mut self) -> Vec { + // Build groups on-the-fly using disk iterator (streams one leaf at a time) + let mut groups_map: IndexMap = IndexMap::new(); + let mut iters: Vec<(XtreamCluster, Box + Send>)> = vec![]; + if let Some((q, _)) = self.live.as_mut() { + iters.push((XtreamCluster::Live, Box::new(q.iter().map(|(_, item)| item)))); + } + if let Some((q, _)) = self.vod.as_mut() { + iters.push((XtreamCluster::Video, Box::new(q.iter().map(|(_, item)| item)))); + } + if let Some((q, _)) = self.series.as_mut() { + iters.push((XtreamCluster::Series, Box::new(q.iter().map(|(_, item)| item)))); + } + + for (cluster, iter) in iters { + for item in iter { + groups_map.entry(item.category_id) + .or_insert_with(|| PlaylistGroup { + id: item.category_id, + title: item.group.clone(), + channels: vec![], + xtream_cluster: cluster, + }) + .channels.push(PlaylistItem::from(&item)); + } + } + + // Sort channels within each group + for group in groups_map.values_mut() { + group.channels.sort_by_key(|item| item.header.source_ordinal); + } + + // Sort groups based on the source_ordinal of their first channel + let mut groups: Vec = groups_map.into_values().collect(); + groups.sort_by_key(|group| { + group.channels.first().map_or(u32::MAX, |c| c.header.source_ordinal) + }); + groups + } + + fn clone_box(&self) -> Box { + let live_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Live); + let vod_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Video); + let series_path = xtream_get_file_path(&self.storage_path, XtreamCluster::Series); + + let live = self.live.as_ref().and_then(|(_, guard)| { + BPlusTreeQuery::try_new(&live_path).ok().map(|q| (q, Arc::clone(guard))) + }); + let vod = self.vod.as_ref().and_then(|(_, guard)| { + BPlusTreeQuery::try_new(&vod_path).ok().map(|q| (q, Arc::clone(guard))) + }); + let series = self.series.as_ref().and_then(|(_, guard)| { + BPlusTreeQuery::try_new(&series_path).ok().map(|q| (q, Arc::clone(guard))) + }); + + Box::new(Self { + app_config: Arc::clone(&self.app_config), + storage_path: self.storage_path.clone(), + live, + vod, + series, + }) + } + + fn release_resources(&mut self, cluster: XtreamCluster) { + match cluster { + XtreamCluster::Live => self.live = None, + XtreamCluster::Video => self.vod = None, + XtreamCluster::Series => self.series = None, + } + } + + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { + Box::pin(async move { + self.reload().await; + }) + } + fn sort_by_provider_ordinal(&mut self) { + warn!("Sorting by provider ordinal is not supported for disk based playlists"); + } +} + +macro_rules! impl_single_file_disk_source { + ($name:ident, $key_type:tt, $entry_type:tt) => { + paste::paste! { + pub struct [<$name DiskPlaylistSource>] { + app_config: Arc, + file_path: PathBuf, + playlist: Option>, + guard: Option>, + } + + impl [<$name DiskPlaylistSource>] { + pub(crate) async fn new(app_config: &Arc, file_path: &Path) -> Self { + let mut source = Self { + app_config: Arc::clone(app_config), + file_path: file_path.to_path_buf(), + playlist: None, + guard: None, + }; + source.reload().await; + source + } + + async fn reload(&mut self) { + self.guard = None; + self.playlist = load_bplustree_query::<$key_type, $entry_type>(&self.app_config, &self.file_path).await + .map(|(query, guard)| { + self.guard = Some(Arc::new(guard)); + query + }); + } + } + + impl PlaylistSource for [<$name DiskPlaylistSource>] { + + fn get_channel_count(&mut self) -> usize { self.playlist.as_mut().map_or(0usize, |t: &mut BPlusTreeQuery<$key_type, $entry_type>| t.len().unwrap_or(0usize)) } + + fn is_memory(&self) -> bool { false } + + fn get_group_count(&mut self) -> usize { + let mut groups = HashSet::new(); + if let Some(query) = self.playlist.as_mut() { for (_, item) in query.iter() { groups.insert(item.group.clone()); } } + groups.len() + } + + fn is_empty(&mut self) -> bool { self.playlist.as_mut().map_or(true, |t| t.is_empty().unwrap_or(true)) } + + fn into_items(&mut self) -> Box + Send + '_> { + if let Some(q) = self.playlist.as_mut() { + Box::new(q.iter().map(|(_, item)| PlaylistItem::from(&item))) + } else { + Box::new(std::iter::empty()) + } + } + + + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + if let Some(pl) = self.playlist.as_mut() { + let iter = pl.iter().map(|(_, item)| Cow::Owned(PlaylistItem::from(&item))); + Box::new(iter) + } else { + Box::new(std::iter::empty()) + } + } + + fn items_mut(&mut self) -> Box + Send + '_> { + warn!("Disk-based playlist sources are read-only. Use clone_source() and convert to memory for mutable access."); + Box::new(std::iter::empty()) + } + + fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { + Box::pin(async move { + warn!("update_playlist should not be called for M3U Disk playlist"); + }) + } + + fn get_missing_vod_info_count(&mut self) -> usize { 0 } + fn get_missing_series_info_count(&mut self) -> usize { 0 } + fn deduplicate(&mut self, _duplicates: &mut HashSet) { + warn!("Deduplication is not supported for disk based playlist updates"); + } + fn take_groups(&mut self) -> Vec { + // Build groups on-the-fly using disk iterator (streams one leaf at a time) + if let Some(q) = self.playlist.as_mut() { + let mut groups_map: IndexMap, PlaylistGroup> = IndexMap::new(); + for (_, item) in q.iter() { + groups_map.entry(item.group.clone()) + .or_insert_with(|| PlaylistGroup { + id: 0, + title: item.group.clone(), + channels: vec![], + xtream_cluster: XtreamCluster::try_from(item.item_type).unwrap_or(XtreamCluster::Live), + }) + .channels.push(PlaylistItem::from(&item)); + } + // Sort channels within each group + for group in groups_map.values_mut() { + group.channels.sort_by_key(|item| item.header.source_ordinal); + } + + // Sort groups based on the source_ordinal of their first channel + let mut groups: Vec = groups_map.into_values().collect(); + groups.sort_by_key(|group| { + group.channels.first().map_or(u32::MAX, |c| c.header.source_ordinal) + }); + groups + } else { + vec![] + } + } + fn clone_box(&self) -> Box { + let playlist = if self.playlist.is_some() && self.guard.is_some() { + BPlusTreeQuery::try_new(&self.file_path).ok() + } else { None }; + + Box::new(Self { + app_config: Arc::clone(&self.app_config), + file_path: self.file_path.clone(), + playlist, + guard: self.guard.clone(), + }) + } + + fn release_resources(&mut self, _cluster: XtreamCluster) { + self.guard = None; + self.playlist = None; + } + + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { + Box::pin(async move { + self.reload().await; + }) + } + + fn sort_by_provider_ordinal(&mut self) { + warn!("Sorting by provider ordinal is not supported for disk based playlists"); + } + } + } + }; +} + +impl_single_file_disk_source!(M3u, String, M3uPlaylistItem); + +impl_single_file_disk_source!(LocalLibrary, UUIDType, XtreamPlaylistItem); + +pub struct MemoryPlaylistSource { + playlist: Arc>, +} + +impl MemoryPlaylistSource { + pub(crate) fn new(groups: Vec) -> Self { + Self { playlist: Arc::new(groups) } + } + + pub fn boxed(self) -> Box { + Box::new(self) + } +} + +impl Default for MemoryPlaylistSource { + fn default() -> Self { + Self { playlist: Arc::new(vec![]) } + } +} + +impl PlaylistSource for MemoryPlaylistSource { + fn is_memory(&self) -> bool { true } + fn get_channel_count(&mut self) -> usize { self.playlist.iter().map(|group| group.channels.len()).sum() } + fn get_group_count(&mut self) -> usize { self.playlist.len() } + fn is_empty(&mut self) -> bool { self.playlist.is_empty() } + fn into_items(&mut self) -> Box + Send + '_> { + let playlist = Arc::make_mut(&mut self.playlist); + Box::new(playlist.iter_mut().flat_map(|group| group.channels.drain(..))) + } + fn items_mut(&mut self) -> Box + Send + '_> { + let playlist = Arc::make_mut(&mut self.playlist); + Box::new(playlist.iter_mut().flat_map(|group| group.channels.iter_mut())) + } + + fn items<'a>(&'a mut self) -> Box> + Send + 'a> { + Box::new( + self.playlist.iter() + .flat_map(|group| group.channels.iter().map(Cow::Borrowed)) + ) + } + + fn update_playlist<'a>(&'a mut self, plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { + Box::pin(async move { + let playlist = Arc::make_mut(&mut self.playlist); + for grp in playlist.iter_mut() { + if grp.id == plg.id { + grp.channels.extend(plg.channels.iter().cloned()); + return; + } + } + playlist.push(plg.clone()); + }) + } + fn get_missing_vod_info_count(&mut self) -> usize { + self.playlist.iter() + .flat_map(|plg| &plg.channels) + .filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video + && pli.header.item_type == PlaylistItemType::Video + && !pli.has_details()).count() + } + fn get_missing_series_info_count(&mut self) -> usize { + self.playlist.iter() + .flat_map(|plg| &plg.channels) + .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Series + && pli.header.item_type == PlaylistItemType::SeriesInfo + && pli.get_provider_id().is_some_and(|id| id > 0) + && !pli.has_details()).count() + } + fn deduplicate(&mut self, duplicates: &mut HashSet) { + let playlist = Arc::make_mut(&mut self.playlist); + for group in playlist { + group.channels.retain(|item| duplicates.insert(item.get_uuid())); + } + } + fn take_groups(&mut self) -> Vec { + std::mem::take(Arc::make_mut(&mut self.playlist)) + } + fn clone_box(&self) -> Box { + Box::new(MemoryPlaylistSource { playlist: Arc::clone(&self.playlist) }) + } + fn release_resources(&mut self, _cluster: XtreamCluster) { /* noop */ } + fn obtain_resources(&mut self) -> BoxFuture<'_, ()> { Box::pin(async move {}) } + + fn sort_by_provider_ordinal(&mut self) { + let playlist = Arc::make_mut(&mut self.playlist); + for group in &mut *playlist { + group.channels.sort_by_key(|item| item.header.source_ordinal); + } + playlist.sort_by_key(|group| + group.channels.first().map_or(u32::MAX, |c| c.header.source_ordinal) + ); + } +} + +async fn load_bplustree_query(app_config: &Arc, file_path: &Path) -> Option<(BPlusTreeQuery, FileReadGuard)> +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + P: Serialize + for<'de> Deserialize<'de> + Clone + Send + 'static, +{ + if file_path.exists() { + let guard = app_config.file_locks.read_lock(file_path).await; + match BPlusTreeQuery::::try_new(file_path) { + Ok(query) => Some((query, guard)), + Err(err) => { + error!("Error loading disk playlist {}: {err}", file_path.display()); + None + } + } + } else { None } +} \ No newline at end of file diff --git a/backend/src/repository/storage.rs b/backend/src/repository/storage.rs index 9ead7b5fc..7e242cb06 100644 --- a/backend/src/repository/storage.rs +++ b/backend/src/repository/storage.rs @@ -1,9 +1,10 @@ -use std::path::{Path, PathBuf}; -use shared::error::{TuliproxError}; -use crate::model::{Config}; -use shared::error::{notify_err}; +use crate::model::Config; use crate::repository::storage_const; use crate::utils; +use shared::error::notify_err; +use shared::error::TuliproxError; +use std::path::{Path, PathBuf}; +use shared::notify_err_res; pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { // Join directly with &str to avoid an intermediate PathBuf allocation @@ -14,12 +15,12 @@ pub fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result std::io::R let sanitized_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); - let name = format!("input_{sanitized_name}"); + let name = format!("input_{sanitized_name}"); let path = Path::new(working_dir).join(name); // Create the directory and return the path or propagate the error std::fs::create_dir_all(&path).map(|()| path) } +pub fn ensure_input_storage_path(cfg: &Config, input_name: &str) -> Result { + get_input_storage_path(input_name, &cfg.working_dir) + .map_err(|err| { + notify_err!("Failed to save input data, can't create directory for input {input_name}: {err}") + }) +} + + pub fn get_geoip_path(working_dir: &str) -> PathBuf { Path::new(working_dir).join("geoip.db") } \ No newline at end of file diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 85bf85782..1e9230140 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -5,16 +5,16 @@ use crate::model::{ApiProxyServerInfo, AppConfig, ProxyUserCredentials}; use crate::model::{ConfigTarget, StrmTargetOutput}; use crate::repository::storage::{ensure_target_storage_path}; use crate::repository::storage_const; -use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, IO_BUFFER_SIZE}; +use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, + IO_BUFFER_SIZE}; use chrono::Datelike; use filetime::{set_file_times, FileTime}; use log::{error, trace}; use regex::Regex; use serde::Serialize; -use shared::error::{create_tuliprox_error_result, info_err}; -use shared::error::{TuliproxError, TuliproxErrorKind}; +use shared::error::{TuliproxError, info_err_res, info_err}; use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType}; -use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; +use shared::utils::{ arc_str_serde, extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -130,13 +130,15 @@ pub fn strm_get_file_paths(file_prefix: &str, target_path: &Path) -> PathBuf { #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct StrmItemInfo { - group: String, + #[serde(with = "arc_str_serde")] + group: Arc, title: String, item_type: PlaylistItemType, #[serde(default, skip_serializing_if = "Option::is_none")] provider_id: Option, virtual_id: u32, - input_name: String, + #[serde(with = "arc_str_serde")] + input_name: Arc, url: String, #[serde(default, skip_serializing_if = "is_blank_optional_string")] series_name: Option, @@ -219,10 +221,7 @@ async fn prepare_strm_output_directory(path: &Path) -> Result<(), TuliproxError> // Ensure the directory exists if let Err(e) = tokio::fs::create_dir_all(path).await { error!("Failed to create directory {}: {e}", path.display()); - return create_tuliprox_error_result!( - TuliproxErrorKind::Notify, - "Error creating STRM directory: {e}" - ); + return info_err_res!("Error creating STRM directory: {e}"); } Ok(()) } @@ -713,10 +712,7 @@ pub async fn write_strm_playlist( &config.working_dir, Some(std::path::PathBuf::from(&target_output.directory)), ) else { - return Err(info_err!(format!( - "Failed to get file path for {}", - target_output.directory - ))); + return info_err_res!("Failed to get file path for {}",target_output.directory); }; let user_and_server_info = get_credentials_and_server_info(app_config, target_output.username.as_deref()); @@ -801,7 +797,7 @@ pub async fn write_strm_playlist( if failed.is_empty() { Ok(()) } else { - Err(info_err!(failed.join(", "))) + Err(info_err!("{}", failed.join(", "))) } } async fn write_strm_index_file( diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index b5327281f..18fc1db99 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -5,8 +5,7 @@ use crate::repository::user_repository::user_get_bouquet_filter; use crate::repository::xtream_repository::{xtream_get_file_path, xtream_get_storage_path}; use crate::utils::FileReadGuard; use log::error; -use shared::error::info_err; -use shared::error::TuliproxError; +use shared::error::{TuliproxError, info_err, info_err_res}; use shared::model::{PlaylistItemType, TargetType, XtreamCluster, XtreamMappingOptions, XtreamPlaylistItem}; use std::collections::HashSet; @@ -31,17 +30,17 @@ impl XtreamPlaylistIterator { // TODO use playlist memory cache and keep sorted - let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name)))?; + let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected: xtream output required for target {}", target.name))?; let config = app_config.config.load(); if let Some(storage_path) = xtream_get_storage_path(&config, target.name.as_str()) { let xtream_path = xtream_get_file_path(&storage_path, cluster); if !xtream_path.exists() { - return Err(info_err!(format!("No {cluster} entries found for target {}", &target.name))); + return info_err_res!("No {cluster} entries found for target {}", &target.name); } let file_lock = app_config.file_locks.read_lock(&xtream_path).await; let query = BPlusTreeQuery::::try_new(&xtream_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {xtream_path:?} - {err}")))?; + .map_err(|err| info_err!("Could not open BPlusTreeQuery {xtream_path:?} - {err}"))?; let reader = query.disk_iter(); let server_info = app_config.get_user_server_info(user); @@ -67,7 +66,7 @@ impl XtreamPlaylistIterator { lookup_item: None, }) } else { - Err(info_err!(format!("Failed to find xtream storage for target {}", &target.name))) + info_err_res!("Failed to find xtream storage for target {}", &target.name) } } diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index 711602947..6702069e3 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -17,21 +17,20 @@ use indexmap::IndexMap; use log::error; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use shared::error::{create_tuliprox_error, create_tuliprox_error_result, info_err, notify_err, string_to_io_error, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, info_err, notify_err, string_to_io_error, TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemType, SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::get_u32_from_serde_value; -use std::borrow::Cow; +use shared::utils::{arc_str_serde, get_u32_from_serde_value, intern, StringInterner}; use std::collections::HashMap; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use shared::notify_err_res; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_tuliprox_error!( - TuliproxErrorKind::Notify, + notify_err!( "failed to write xtream playlist: {} - {}", $path.display(), $err @@ -40,39 +39,38 @@ macro_rules! cant_write_result { } #[inline] -fn get_collection_path(path: &Path, collection: &str) -> PathBuf { +pub fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } #[inline] -fn get_live_cat_collection_path(path: &Path) -> PathBuf { +pub fn get_live_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_LIVE) } #[inline] -fn get_vod_cat_collection_path(path: &Path) -> PathBuf { +pub fn get_vod_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_VOD) } #[inline] -fn get_series_cat_collection_path(path: &Path) -> PathBuf { +pub fn get_series_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_SERIES) } - -fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { +pub fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { if let Some(path) = xtream_get_storage_path(cfg, target_name) { if std::fs::create_dir_all(&path).is_err() { let msg = format!( "Failed to save xtream data, can't create directory {}", &path.display() ); - return Err(notify_err!(msg)); + return notify_err_res!("{msg}"); } Ok(path) } else { let msg = format!("Failed to save xtream data, can't create directory for target {target_name}"); - Err(notify_err!(msg)) + notify_err_res!("{msg}") } } @@ -167,9 +165,11 @@ fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Optio None } -type CategoryKey = (XtreamCluster, Cow<'static, str>); +pub type CategoryKey = (XtreamCluster, Arc); -async fn load_old_category_ids(path: &Path) -> (u32, HashMap) { +// Because interner is not thread safe we cant use it currently for interning. +// We leave the argument for later optimizations. +async fn load_old_category_ids(path: &Path, _interner: &mut StringInterner) -> (u32, HashMap) { let old_path = path.to_path_buf(); tokio::task::spawn_blocking(move || { let mut result: HashMap = HashMap::new(); @@ -190,7 +190,7 @@ async fn load_old_category_ids(path: &Path) -> (u32, HashMap) if let Some(category_id) = entry.get(crate::model::XC_TAG_CATEGORY_ID).and_then(get_u32_from_serde_value) { if let Value::Object(item) = entry { if let Some(category_name) = get_map_item_as_str(&item, crate::model::XC_TAG_CATEGORY_NAME) { - result.insert((cluster, Cow::Owned(category_name)), category_id); + result.insert((cluster, /*interner.*/intern(&category_name)), category_id); max_id = max_id.max(category_id); } } @@ -226,16 +226,18 @@ pub fn xtream_get_file_path(storage_path: &Path, cluster: XtreamCluster) -> Path } #[derive(Serialize, Deserialize)] -struct CategoryEntry { - category_id: u32, - category_name: String, - parent_id: u32, +pub struct CategoryEntry { + pub category_id: u32, + #[serde(with = "arc_str_serde")] + pub category_name: Arc, + pub parent_id: u32, } pub async fn xtream_write_playlist( app_cfg: &Arc, target: &ConfigTarget, playlist: &mut [PlaylistGroup], + interner: &mut StringInterner, ) -> Result<(), TuliproxError> { let path = { let config = app_cfg.config.load(); @@ -249,7 +251,7 @@ pub async fn xtream_write_playlist( let mut series_col = Vec::with_capacity(50_000); let mut vod_col = Vec::with_capacity(50_000); - let categories = create_categories(playlist, &path).await; + let categories = create_categories(playlist, &path, interner).await; { for (xtream_cluster, category) in categories { match xtream_cluster { @@ -306,25 +308,21 @@ pub async fn xtream_write_playlist( } if !errors.is_empty() { - return create_tuliprox_error_result!( - TuliproxErrorKind::Notify, - "{}", - errors.join("\n") - ); + return info_err_res!("{}", errors.join("\n")); } Ok(()) } -async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<(XtreamCluster, CategoryEntry)> { +async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path, interner: &mut StringInterner) -> Vec<(XtreamCluster, CategoryEntry)> { // preserve category_ids - let (max_cat_id, existing_cat_ids) = load_old_category_ids(path).await; + let (max_cat_id, existing_cat_ids) = load_old_category_ids(path, interner).await; let mut cat_id_counter = max_cat_id; - let mut new_categories: IndexMap<(XtreamCluster, Cow<'_, str>), CategoryEntry> = IndexMap::new(); + let mut new_categories: IndexMap = IndexMap::new(); let mut last_cluster: Option = None; - let mut last_group = String::new(); + let mut last_group = intern(""); let mut last_category_id: u32 = 0; for plg in playlist.iter_mut() { @@ -334,15 +332,15 @@ async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<( for channel in &mut plg.channels { let cluster = channel.header.xtream_cluster; - let group = channel.header.group.as_str(); + let group = &channel.header.group; // Fast path - if last_cluster == Some(cluster) && last_group == group { + if last_cluster == Some(cluster) && &last_group == group { channel.header.category_id = last_category_id; continue; } - let key = (cluster, Cow::Borrowed(group)); + let key = (cluster, Arc::clone(group)); let entry = new_categories.entry(key.clone()).or_insert_with(|| { let cat_id = existing_cat_ids.get(&key).copied().unwrap_or_else(|| { @@ -352,14 +350,13 @@ async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<( CategoryEntry { category_id: cat_id, - category_name: group.to_string(), + category_name: group.clone(), parent_id: 0, } }); last_cluster = Some(cluster); - last_group.clear(); - last_group.push_str(group); + last_group = Arc::clone(group); last_category_id = entry.category_id; channel.header.category_id = last_category_id; @@ -584,7 +581,7 @@ pub async fn iter_raw_xtream_playlist(app_config: &AppConfig, target: &ConfigTar } let file_lock = app_config.file_locks.read_lock(&xtream_path).await; match BPlusTreeQuery::::try_new(&xtream_path) - .map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {xtream_path:?} - {err}"))) { + .map_err(|err| info_err!("Could not open BPlusTreeQuery {xtream_path:?} - {err}")) { Ok(mut query) => { let items: Vec = query.iter().map(|(_, v)| v).collect(); let len = items.len(); @@ -757,7 +754,7 @@ pub async fn persist_input_xtream_playlist(app_config: &Arc, storage_ let err = if errors.is_empty() { None } else { - Some(create_tuliprox_error!(TuliproxErrorKind::Notify, "{}", errors.join("\n"))) + Some(notify_err!("{}", errors.join("\n"))) }; (result, err) @@ -802,6 +799,41 @@ async fn persist_input_info(app_config: &Arc, storage_path: &Path, cl Ok(()) } +pub async fn persist_input_info_batch(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, + input_name: &str, updates: Vec<(u32, StreamProperties)>) -> Result<(), Error> { + if updates.is_empty() { return Ok(()); } + let xtream_path = xtream_get_file_path(storage_path, cluster); + if xtream_path.exists() { + let _file_lock = app_config.file_locks.write_lock(&xtream_path).await; + let mut tree: BPlusTreeUpdate = BPlusTreeUpdate::try_new(&xtream_path) + .map_err(|err| Error::other(format!("failed to open BPlusTree for input {input_name}: {err}")))?; + + let mut updated_plis = Vec::with_capacity(updates.len()); + for (provider_id, props) in updates { + match tree.query(&provider_id) { + Ok(Some(mut pli)) => { + pli.additional_properties = Some(props); + updated_plis.push((provider_id, pli)); + } + Ok(None) => { + error!("Could not find input entry for provider_id: {provider_id} and input: {input_name}"); + } + Err(err) => { + error!("Failed to query BPlusTree for provider_id: {provider_id} and input: {input_name}: {err}"); + } + } + } + + if !updated_plis.is_empty() { + let refs: Vec<(&u32, &XtreamPlaylistItem)> = updated_plis.iter() + .map(|(id, pli)| (id, pli)) + .collect(); + tree.update_batch(&refs).map_err(|err| Error::other(format!("failed to write batch {cluster} info for input {input_name}: {err}")))?; + } + } + Ok(()) +} + pub async fn persist_input_vod_info(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, input_name: &str, provider_id: u32, @@ -809,12 +841,30 @@ pub async fn persist_input_vod_info(app_config: &Arc, storage_path: & persist_input_info(app_config, storage_path, cluster, input_name, provider_id, StreamProperties::Video(Box::new(props.clone()))).await } +pub async fn persist_input_vod_info_batch(app_config: &Arc, storage_path: &Path, + cluster: XtreamCluster, input_name: &str, + updates: Vec<(u32, VideoStreamProperties)>) -> Result<(), Error> { + let batch = updates.into_iter() + .map(|(id, props)| (id, StreamProperties::Video(Box::new(props)))) + .collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} + pub async fn persists_input_series_info(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, input_name: &str, provider_id: u32, props: &SeriesStreamProperties) -> Result<(), Error> { persist_input_info(app_config, storage_path, cluster, input_name, provider_id, StreamProperties::Series(Box::new(props.clone()))).await } +pub async fn persist_input_series_info_batch(app_config: &Arc, storage_path: &Path, + cluster: XtreamCluster, input_name: &str, + updates: Vec<(u32, SeriesStreamProperties)>) -> Result<(), Error> { + let batch = updates.into_iter() + .map(|(id, props)| (id, StreamProperties::Series(Box::new(props)))) + .collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} + pub async fn load_input_xtream_playlist(app_config: &Arc, storage_path: &Path, clusters: &[XtreamCluster]) -> Result, TuliproxError> { let mut groups: IndexMap = IndexMap::new(); @@ -851,7 +901,7 @@ pub async fn load_input_xtream_playlist(app_config: &Arc, storage_pat groups.entry(cat_id) .or_insert_with(|| PlaylistGroup { id: cat_id, - title: String::from("Unknown"), + title: intern("Unknown"), channels: Vec::new(), xtream_cluster: cluster, }) diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index afff6c676..9f73999d1 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -11,7 +11,7 @@ use arc_swap::{ArcSwap, ArcSwapAny}; use chrono::Local; use log::{error, info, warn}; use serde::Serialize; -use shared::error::{create_tuliprox_error, info_err, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err, info_err_res, TuliproxError}; use shared::model::{ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, SourcesConfigDto, TargetUserDto}; use shared::utils::CONSTANTS; use std::env; @@ -72,15 +72,15 @@ pub fn read_sources_file_from_path(sources_file: &Path, resolve_env: bool, inclu Ok(mut sources) => { if resolve_env { if let Err(err) = sources.prepare(include_computed, hdhr_config) { - return Err(info_err!(format!("Can't read the sources-config file: {}: {err}", sources_file.display()))); + return info_err_res!("Can't read the sources-config file: {}: {err}", sources_file.display()); } } Ok(sources) } - Err(err) => Err(info_err!(format!("Can't read the sources-config file: {}: {err}", sources_file.display()))) + Err(err) => info_err_res!("Can't read the sources-config file: {}: {err}", sources_file.display()) } } - Err(err) => Err(info_err!(format!("Can't read the sources-config file: {}: {err}", sources_file.display()))) + Err(err) => info_err_res!("Can't read the sources-config file: {}: {err}", sources_file.display()) } } @@ -100,10 +100,10 @@ pub fn read_config_file(config_file: &str, resolve_env: bool, include_computed: } Ok(config) } - Err(err) => Err(info_err!(format!("Can't read the config file: {config_file}: {err}"))) + Err(err) => info_err_res!("Can't read the config file: {config_file}: {err}") } } - Err(err) => Err(info_err!(format!("Can't read the config file: {config_file}: {err}"))) + Err(err) => info_err_res!("Can't read the config file: {config_file}: {err}") } } @@ -175,7 +175,7 @@ pub async fn get_batch_aliases(input_type: InputType, url: &str) -> Result { - Err(TuliproxError::new(TuliproxErrorKind::Info, err.to_string())) + info_err_res!("{err}") } }; } @@ -285,7 +285,7 @@ pub fn read_api_proxy_file(api_proxy_file: &str, resolve_env: bool) -> Result { - Err(info_err!(format!("cant read api-proxy-config file: {err}"))) + info_err_res!("cant read api-proxy-config file: {err}") } } }) @@ -319,8 +319,25 @@ where { let path = PathBuf::from(file_path); let filename = path.file_name().map_or(default_name.to_string(), |f| f.to_string_lossy().to_string()); - let backup_path = PathBuf::from(backup_dir).join(format!("{filename}_{}", Local::now().format("%Y%m%d_%H%M%S"))); + + let mut serialized = String::new(); + let options = serde_saphyr::SerializerOptions { + prefer_block_scalars: false, + ..Default::default() + }; + serde_saphyr::to_fmt_writer_with_options(&mut serialized, &config, options) + .map_err(|err| info_err!("Could not serialize config: {}", err))?; + if path.exists() { + if let Ok(existing) = fs::read_to_string(&path).await { + if existing == serialized { + // info!("File {} unchanged, skipping write", path.display()); + return Ok(()); + } + } + } + + let backup_path = PathBuf::from(backup_dir).join(format!("{filename}_{}", Local::now().format("%Y%m%d_%H%M%S"))); match fs::copy(&path, &backup_path).await { Ok(_) => {} @@ -328,17 +345,9 @@ where } info!("Saving file to {}", &path.to_str().unwrap_or("?")); - let mut serialized = String::new(); - let options = serde_saphyr::SerializerOptions { - prefer_block_scalars: false, - ..Default::default() - }; - serde_saphyr::to_fmt_writer_with_options(&mut serialized, &config, options) - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Info, "Could not serialize config: {}", err))?; - fs::write(&path, serialized) .await - .map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Info, "Could not write file {}: {}", &path.to_str().unwrap_or("?"), err)) + .map_err(|err| info_err!("Could not write file {}: {}", &path.to_str().unwrap_or("?"), err)) } pub async fn save_api_proxy(file_path: &str, backup_dir: &str, config: &ApiProxyConfigDto) -> Result<(), TuliproxError> { diff --git a/backend/src/utils/file/mapping_reader.rs b/backend/src/utils/file/mapping_reader.rs index d3eccbf54..49980c216 100644 --- a/backend/src/utils/file/mapping_reader.rs +++ b/backend/src/utils/file/mapping_reader.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; use crate::model::{Mappings}; -use shared::error::{create_tuliprox_error_result, info_err, TuliproxError, TuliproxErrorKind}; +use shared::error::{info_err_res, TuliproxError}; use crate::utils::traverse_dir; use crate::utils::{config_file_reader, open_file}; use log::{warn}; use std::path::{Path, PathBuf}; +use shared::info_err; use shared::model::{MappingDefinitionDto, MappingDto, MappingsDto, PatternTemplate}; fn read_mapping(mapping_file: &Path, resolve_var: bool, prepare_mappings: bool) -> Result, TuliproxError> { @@ -18,7 +19,7 @@ fn read_mapping(mapping_file: &Path, resolve_var: bool, prepare_mappings: bool) Ok(Some(mapping)) } Err(err) => { - Err(info_err!(err.to_string())) + info_err_res!("{err}") } }; } @@ -91,9 +92,9 @@ fn read_mappings_from_directory(path: &Path, resolve_env: bool) -> Result