diff --git a/CHANGELOG.md b/CHANGELOG.md index ab3f91924..d4bfc4767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Changelog # 3.3.0 (2025-11-xx) -- !BREAKING CHANGE! Diue to some structure changes the old data format is invalid. You need to clean your data folder and update the playlists +- !BREAKING CHANGE! Due to some structure changes the old data format is invalid. You need to clean your data folder and update the playlists - !BREAKING CHANGE! config.yml threads attribute is now renamed to process_parallel and is a boolean (true or false). - !BREAKING CHANGE! config.yml adds a reverse proxy config field rewrite_secret to keep resource URLs valid after restart. - !BREAKING CHANGE! removed `forced_retry_interval_secs`. +- !BREAKING CHANGE! `name` attribute is mandatory for input type batch. The name attribute is used for playlist `uuid` and needs to be stable. The first alias is renamed with the input `name` attribute. - Avoid blocking the runtime when warming the cache. - Normalize FileLockManager paths so aliases share the same lock. - Use async file operations for playlist persistence to avoid blocking the async runtime. diff --git a/Cargo.lock b/Cargo.lock index 4bca91e3f..e633f44cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,24 +258,19 @@ dependencies = [ ] [[package]] -name = "bincode" -version = "2.0.1" +name = "bit-set" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bincode_derive", - "serde", - "unty", + "bit-vec", ] [[package]] -name = "bincode_derive" -version = "2.0.1" +name = "bit-vec" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -355,6 +350,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.0" @@ -995,6 +996,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1108,7 +1120,7 @@ dependencies = [ "futures-signals", "gloo-render 0.2.0", "gloo-storage 0.3.0", - "gloo-timers 0.2.6", + "gloo-timers 0.3.0", "gloo-utils 0.2.0", "implicit-clone", "js-sys", @@ -1621,8 +1633,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" dependencies = [ - "futures-channel", - "futures-core", "js-sys", "wasm-bindgen", ] @@ -1672,7 +1682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13471584da78061a28306d1359dd0178d8d6fc1c7c80e5e35d27260346e0516a" dependencies = [ "anymap2", - "bincode 1.3.3", + "bincode", "gloo-console 0.2.3", "gloo-utils 0.1.7", "js-sys", @@ -1688,7 +1698,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76495d3dd87de51da268fa3a593da118ab43eb7f8809e17eb38d3319b424e400" dependencies = [ - "bincode 1.3.3", + "bincode", "futures", "gloo-utils 0.2.0", "gloo-worker-macros", @@ -1707,7 +1717,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "085f262d7604911c8150162529cefab3782e91adb20202e8658f7275d2aefe5d" dependencies = [ - "bincode 1.3.3", + "bincode", "futures", "gloo-utils 0.2.0", "gloo-worker-macros", @@ -1825,6 +1835,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hkdf" version = "0.12.4" @@ -3415,6 +3431,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "route-recognizer" version = "0.3.1" @@ -3803,6 +3841,7 @@ dependencies = [ "ciborium", "enum-iterator", "fastrand", + "hex", "indexmap", "js-sys", "log", @@ -3813,6 +3852,7 @@ dependencies = [ "serde", "serde_json", "url", + "uuid", "zeroize", ] @@ -4362,7 +4402,6 @@ dependencies = [ "async-compression", "axum", "base64", - "bincode 2.0.1", "blake3", "bytes", "chrono", @@ -4374,6 +4413,7 @@ dependencies = [ "deunicode", "enum-iterator", "env_logger", + "fancy-regex", "fastrand", "filetime", "flate2", @@ -4386,6 +4426,7 @@ dependencies = [ "log", "lol_html", "mime", + "mime_guess", "notify", "openssl", "paste", @@ -4397,6 +4438,7 @@ dependencies = [ "rayon", "regex", "reqwest", + "rmp-serde", "rpassword", "rphonetic", "rust-argon2", @@ -4481,12 +4523,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - [[package]] name = "url" version = "2.5.7" @@ -4570,12 +4606,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - [[package]] name = "walkdir" version = "2.5.0" diff --git a/README.md b/README.md index 2cece4210..5b5c656f5 100644 --- a/README.md +++ b/README.md @@ -684,13 +684,6 @@ library: formats: - "nfo" # Optionally write Kodi-compatible NFO files - classification: - series_patterns: - - "(?i)s\\d+e\\d+" # S01E02 format - - "(?i)\\d+x\\d+" # 1x02 format - - "(?i)season[\\s._-]*\\d+" # "Season 1" format - - "(?i)episode[\\s._-]*\\d+" # "Episode 1" format - playlist: movie_category: "Local Movies" series_category: "Local Series" @@ -930,6 +923,7 @@ There are 2 batch input types `xtream_batch` and `m3u_batch`. - sources: - inputs: - type: xtream_batch + name: my_provider url: 'file:///home/tuliprox/config/my_provider_batch.csv' targets: - name: test @@ -940,6 +934,8 @@ There are 2 batch input types `xtream_batch` and `m3u_batch`. my_provider_1;user1;password1;http://my_provider_1.com:80;1;0;2028-11-23 12:34:23 my_provider_2;user2;password2;http://my_provider_2.com:8080;1;0;2028-11-23 12:34:23 ``` +Important: the first alias is renamed with the `name` from input definition. `my_provider_1` gets `my_provider`. +This is necessary because of playlist uuid generation and assigning same channel numbers on each update. ##### `M3uBatch` ```yaml diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b7d5738f5..7f8bf4582 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -35,7 +35,8 @@ deunicode = "1.6.2" mime = "0.3" log = "0.4" env_logger = "0.11" -bincode = { version = "2.0", features = ["std", "serde"] } +#bincode = { version = "2.0", features = ["std", "serde"] } +rmp-serde = "1" rand = "0.9" fastrand = "2" rpassword = "7.4" @@ -74,6 +75,8 @@ chrono-tz = "0.10.4" zeroize = "1.8.2" sysinfo = { version = "0.37.2", default-features = false, features = ["system"] } uuid = { version = "1", features = ["v4"] } +fancy-regex = "0.17.0" +mime_guess = "2" # [target.'cfg(not(target_env = "msvc"))'.dependencies] # tikv-jemallocator = { version = "*", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index f14fd7f50..83f9984a3 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -1,3 +1,4 @@ +use crate::BUILD_TIMESTAMP; use crate::api::endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext}; use crate::api::model::{ create_channel_unavailable_stream, create_custom_video_stream_response, @@ -13,10 +14,12 @@ use crate::api::panel_api::try_provision_account_on_exhausted; use crate::model::{ConfigInput, ResourceRetryConfig}; use crate::model::{ConfigTarget, ProxyUserCredentials}; use crate::tools::lru_cache::LRUResourceCache; -use crate::utils::request; +use crate::utils::{request}; use crate::utils::{async_file_reader, async_file_writer, create_new_file_for_write, get_file_extension}; use crate::utils::{debug_if_enabled, trace_if_enabled}; -use crate::BUILD_TIMESTAMP; +use crate::auth::Fingerprint; +use crate::utils::request::{content_type_from_ext, parse_range}; + use arc_swap::ArcSwapOption; use axum::http::{header, HeaderMap, HeaderValue, Response, StatusCode}; use axum::response::IntoResponse; @@ -125,8 +128,6 @@ pub use try_option_bad_request; pub use try_result_bad_request; pub use try_result_not_found; pub use try_unwrap_body; -use crate::auth::Fingerprint; -use crate::utils::request::{content_type_from_ext, parse_range}; pub fn get_server_time() -> String { chrono::offset::Local::now() @@ -144,7 +145,7 @@ pub fn get_build_time() -> Option { } #[allow(clippy::missing_panics_doc)] -pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoResponse + Send { +pub async fn serve_file(file_path: &Path, mime_type: String) -> impl IntoResponse + Send { match tokio::fs::try_exists(file_path).await { Ok(exists) => { if !exists { @@ -165,7 +166,7 @@ pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoRes try_unwrap_body!(axum::response::Response::builder() .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, mime_type.to_string()) + .header(axum::http::header::CONTENT_TYPE, mime_type) .header( axum::http::header::CACHE_CONTROL, axum::http::header::HeaderValue::from_static("no-cache") @@ -1208,25 +1209,39 @@ pub fn get_headers_from_request( fn get_add_cache_content( res_url: &str, + mime_type: Option, cache: &Arc>>, ) -> Arc { let resource_url = String::from(res_url); let cache = Arc::clone(cache); let add_cache_content: Arc = Arc::new(move |size| { let res_url = resource_url.clone(); - + let mime_type = mime_type.clone(); // todo spawn, replace with unboundchannel let cache = Arc::clone(&cache); tokio::spawn(async move { if let Some(cache) = cache.load().as_ref() { - let _ = cache.lock().await.add_content(&res_url, size); + let _ = cache.lock().await.add_content(&res_url, mime_type, size); } }); }); add_cache_content } -async fn build_stream_response( +fn get_mime_type(headers: &axum::http::HeaderMap, resource_url: &str) -> Option { + headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) // Option<&str> + .map(ToString::to_string) // Option + .or_else(|| { + // fallback auf guess + mime_guess::from_path(resource_url) + .first_raw() + .map(ToString::to_string) + }) +} + +async fn build_resource_stream_response( app_state: &Arc, resource_url: &str, response: reqwest::Response, @@ -1235,6 +1250,7 @@ async fn build_stream_response( let status = response.status(); let mut response_builder = axum::response::Response::builder().status(status); + let mime_type = get_mime_type(response.headers(), resource_url); let has_content_range = response.headers().contains_key(axum::http::header::CONTENT_RANGE); for (key, value) in response.headers() { let name = key.as_str(); @@ -1261,7 +1277,7 @@ async fn build_stream_response( if can_cache { debug!( "Caching eligible resource stream {sanitized_resource_url}"); let cache_resource_path = if let Some(cache) = app_state.cache.load().as_ref() { - Some(cache.lock().await.store_path(resource_url)) + Some(cache.lock().await.store_path(resource_url, mime_type.as_deref())) } else { None }; @@ -1270,7 +1286,7 @@ async fn build_stream_response( Ok(file) => { debug!("Persisting resource stream {sanitized_resource_url} to {}", resource_path.display()); let writer = async_file_writer(file); - let add_cache_content = get_add_cache_content(resource_url, &app_state.cache); + let add_cache_content = get_add_cache_content(resource_url, mime_type, &app_state.cache); let tee = tee_stream(byte_stream, writer, &resource_path, add_cache_content); return try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(tee))); } @@ -1313,7 +1329,7 @@ async fn fetch_resource_with_retry( let status = response.status(); if status.is_success() { return Some( - build_stream_response(app_state, resource_url, response).await, + build_resource_stream_response(app_state, resource_url, response).await, ); } // Retry only for 408, 425, 429 and all 5xx statuses @@ -1405,9 +1421,9 @@ pub async fn resource_response( let req_headers = get_headers_from_request(req_headers, &filter); if let Some(cache) = app_state.cache.load().as_ref() { let mut guard = cache.lock().await; - if let Some(resource_path) = guard.get_content(resource_url) { + if let Some((resource_path, mime_type)) = guard.get_content(resource_url) { trace_if_enabled!("Responding resource from cache {}", sanitize_sensitive_info(resource_url)); - return serve_file(&resource_path, mime::APPLICATION_OCTET_STREAM) + return serve_file(&resource_path, mime_type.unwrap_or_else(|| mime::APPLICATION_OCTET_STREAM.to_string())) .await .into_response(); } diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index 59a5516c7..15300021a 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -149,8 +149,8 @@ pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cf Some(input) => { let (result, errors) = match input.input_type { - InputType::M3u | InputType::M3uBatch => m3u::get_m3u_playlist(client, cfg, input).await, - InputType::Xtream | InputType::XtreamBatch => xtream::get_xtream_playlist(cfg, client, input).await, + InputType::M3u | InputType::M3uBatch => m3u::download_m3u_playlist(client, cfg, input).await, + InputType::Xtream | InputType::XtreamBatch => xtream::download_xtream_playlist(cfg, client, input).await, InputType::Library => { return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({ "error": "Library inputs are not supported on this endpoint"}))).into_response(); } diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index 1b815fac1..b476f0d2e 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -13,7 +13,7 @@ use shared::utils::{concat_path_leading_slash}; use std::collections::{BTreeMap, HashMap}; use std::io::{Cursor}; use std::sync::Arc; -use log::error; +use log::{error, info}; use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader; use crate::api::endpoints::v1_api_config::v1_api_config_register; use crate::api::endpoints::library_api::library_api_register; @@ -106,6 +106,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State { + info!("GeoIp db updated"); app_state.geoip.store(Some(Arc::new(geoip))); axum::http::StatusCode::OK.into_response() }, diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index b2d79da51..9f6d34443 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -80,7 +80,6 @@ async fn playlist_update( } } - async fn playlist_content( ExtractAcceptHeader(accept): ExtractAcceptHeader, axum::extract::State(app_state): axum::extract::State>, diff --git a/backend/src/api/endpoints/web_index.rs b/backend/src/api/endpoints/web_index.rs index c16d11edd..1a577f586 100644 --- a/backend/src/api/endpoints/web_index.rs +++ b/backend/src/api/endpoints/web_index.rs @@ -219,7 +219,7 @@ async fn index( error!("Failed to read web ui index.html: {err}"); } } - serve_file(&path, mime::TEXT_HTML_UTF_8).await.into_response() + serve_file(&path, mime::TEXT_HTML_UTF_8.to_string()).await.into_response() } async fn index_config( @@ -279,7 +279,7 @@ async fn index_config( } } } - serve_file(&path, mime::APPLICATION_JSON).await.into_response() + serve_file(&path, mime::APPLICATION_JSON.to_string()).await.into_response() } pub fn index_register_without_path(web_dir_path: &Path) -> axum::Router> { diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index dc3f86124..6701d47d2 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -174,7 +174,7 @@ pub async fn serve_epg( serve_epg_with_rewrites(epg_path, timeshift, rewrite_urls, &encrypt_secret, &base_url, filter).await } else { // Neither timeshift nor rewrite needed, serve original file - serve_file(epg_path, mime::TEXT_XML).await.into_response() + serve_file(epg_path, mime::TEXT_XML.to_string()).await.into_response() }; } } diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index dea4a42a1..a5c51a8de 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1,5 +1,4 @@ // https://github.com/tellytv/go.xtream-codes/blob/master/structs.go - use crate::api::api_utils; use crate::api::api_utils::{create_session_fingerprint, local_stream_response, try_unwrap_body}; use crate::api::api_utils::{ @@ -15,7 +14,7 @@ use crate::api::model::UserApiRequest; use crate::api::model::XtreamAuthorizationResponse; use crate::api::model::{create_custom_video_stream_response, CustomVideoStreamType}; use crate::auth::Fingerprint; -use crate::model::{xtream_mapping_option_from_target_options, AppConfig, ConfigTarget}; +use crate::model::{xtream_mapping_option_from_target_options, ConfigTarget}; use crate::model::{Config, ConfigInput}; use crate::model::{InputSource, ProxyUserCredentials}; use crate::repository::playlist_repository::get_target_id_mapping; @@ -33,15 +32,15 @@ use futures::Stream; use log::{debug, error, warn}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use shared::error::create_tuliprox_error_result; +use shared::error::{create_tuliprox_error_result}; use shared::error::info_err; use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind}; -use shared::model::{create_stream_channel_with_type, get_backdrop_path_value, FieldGetAccessor, PlaylistEntry, PlaylistItemType, ProxyType, TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem}; +use shared::model::{create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType, + TargetType, UserConnectionPermission, XtreamCluster}; use shared::utils::{ - extract_extension_from_url, generate_playlist_uuid, get_u32_from_serde_value, hex_encode, - sanitize_sensitive_info, trim_slash, HLS_EXT, + extract_extension_from_url, generate_playlist_uuid, sanitize_sensitive_info, + trim_slash, HLS_EXT, }; -use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt::{Display, Formatter}; use std::path::Path; @@ -565,163 +564,6 @@ async fn xtream_player_api_stream_with_token( } } -fn get_doc_id_and_field_name(input: &str) -> Option<(u32, &str)> { - if let Some(pos) = input.find('_') { - let (number_part, rest) = input.split_at(pos); - let field = &rest[1..]; // cut _ - if let Ok(number) = number_part.parse::() { - return Some((number, field)); - } - } - None -} - -fn get_doc_resource_field_value<'a>( - field: &'a str, - doc: Option<&'a Value>, -) -> Option> { - if let Some(Value::Object(info_data)) = doc { - if field.starts_with(crate::model::XC_PROP_BACKDROP_PATH) { - return get_backdrop_path_value( - field, - info_data.get(crate::model::XC_PROP_BACKDROP_PATH), - ); - } else if let Some(Value::String(url)) = info_data.get(field) { - return Some(Cow::Borrowed(url)); - } - } - None -} - -async fn xtream_get_info_resource_url<'a>( - config: &'a AppConfig, - pli: &'a XtreamPlaylistItem, - target: &'a ConfigTarget, - resource: &'a str, -) -> Result>, serde_json::Error> { - let info_content = match pli.xtream_cluster { - XtreamCluster::Video => xtream_repository::xtream_load_vod_info( - config, - target.name.as_str(), - pli.get_virtual_id(), - ).await, - XtreamCluster::Series => xtream_repository::xtream_load_series_info( - config, - target.name.as_str(), - pli.get_virtual_id(), - ).await, - XtreamCluster::Live => None, - }; - if let Some(content) = info_content { - let doc: Map = serde_json::from_str(&content)?; - let (field, possible_episode_id) = if let Some(field_name_with_episode_id) = - resource.strip_prefix(crate::model::XC_INFO_RESOURCE_PREFIX_EPISODE) - { - if let Some((episode_id, field_name)) = - get_doc_id_and_field_name(field_name_with_episode_id) - { - (field_name, Some(episode_id)) - } else { - return Ok(None); - } - } else { - ( - &resource[crate::model::XC_INFO_RESOURCE_PREFIX.len()..], - None, - ) - }; - let info_doc = match pli.xtream_cluster { - XtreamCluster::Video | XtreamCluster::Series => { - if let Some(episode_id) = possible_episode_id { - get_episode_info_doc(&doc, episode_id) - } else { - doc.get(crate::model::XC_TAG_INFO_DATA) - } - } - XtreamCluster::Live => None, - }; - - if let Some(value) = get_doc_resource_field_value(field, info_doc) { - return Ok(Some(Cow::Owned(value.into_owned()))); - } - } - Ok(None) -} - -fn get_episode_info_doc(doc: &Map, episode_id: u32) -> Option<&Value> { - let episodes = doc.get(crate::model::XC_TAG_EPISODES)?.as_object()?; - for season_episodes in episodes.values() { - if let Value::Array(episode_list) = season_episodes { - for episode in episode_list { - if let Value::Object(episode_doc) = episode { - if let Some(episode_id_value) = episode_doc.get(crate::model::XC_TAG_ID) { - if let Some(doc_episode_id) = get_u32_from_serde_value(episode_id_value) { - if doc_episode_id == episode_id { - return episode_doc.get(crate::model::XC_TAG_INFO_DATA); - } - } - } - } - } - } - } - None -} - -fn get_season_info_doc(doc: &Vec, season_id: u32) -> Option<&Value> { - for season in doc { - if let Value::Object(season_doc) = season { - if let Some(season_id_value) = season_doc.get(crate::model::XC_TAG_ID) { - if let Some(doc_season_id) = get_u32_from_serde_value(season_id_value) { - if doc_season_id == season_id { - return Some(season); - } - } - } - } - } - None -} - -async fn xtream_get_season_resource_url<'a>( - config: &'a AppConfig, - pli: &'a XtreamPlaylistItem, - target: &'a ConfigTarget, - resource: &'a str, -) -> Result>, serde_json::Error> { - let info_content = match pli.xtream_cluster { - XtreamCluster::Series => xtream_repository::xtream_load_series_info( - config, - target.name.as_str(), - pli.get_virtual_id(), - ).await, - XtreamCluster::Video | XtreamCluster::Live => None, - }; - if let Some(content) = info_content { - let doc: Map = serde_json::from_str(&content)?; - - if let Some(field_name_with_season_id) = - resource.strip_prefix(crate::model::XC_SEASON_RESOURCE_PREFIX) - { - if let Some((season_id, field)) = get_doc_id_and_field_name(field_name_with_season_id) { - let seasons_doc = match pli.xtream_cluster { - XtreamCluster::Series => doc.get(crate::model::XC_TAG_SEASONS_DATA), - XtreamCluster::Video | XtreamCluster::Live => None, - }; - - if let Some(Value::Array(seasons)) = seasons_doc { - if let Some(value) = - get_doc_resource_field_value(field, get_season_info_doc(seasons, season_id)) - { - return Ok(Some(Cow::Owned(value.into_owned()))); - } - } - } - } - } - Ok(None) -} - async fn xtream_player_api_resource( req_headers: &HeaderMap, api_req: &UserApiRequest, @@ -761,23 +603,8 @@ async fn xtream_player_api_resource( true, format!("Failed to read xtream item for stream id {req_virtual_id}") ); - let stream_url = if resource.starts_with(crate::model::XC_INFO_RESOURCE_PREFIX) { - try_result_bad_request!(xtream_get_info_resource_url( - &app_state.app_config, - &pli, - &target, - resource - ).await) - } else if resource.starts_with(crate::model::XC_SEASON_RESOURCE_PREFIX) { - try_result_bad_request!(xtream_get_season_resource_url( - &app_state.app_config, - &pli, - &target, - resource - ).await) - } else { - pli.get_field(resource) - }; + + let stream_url = pli.resolve_resource_url(resource); match stream_url { None => axum::http::StatusCode::NOT_FOUND.into_response(), @@ -790,9 +617,7 @@ async fn xtream_player_api_resource( redirect(&url).into_response() } else { trace_if_enabled!("Resource request to {}", sanitize_sensitive_info(&url)); - resource_response(app_state, &url, req_headers, None) - .await - .into_response() + resource_response(app_state, &url, req_headers, None).await.into_response() } } } @@ -1009,9 +834,10 @@ async fn xtream_get_stream_info_response( let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name))) else { return try_unwrap_body!(empty_json_response()); }; + let server_info = app_state.app_config.get_user_server_info(user); let options = xtream_mapping_option_from_target_options(target, xtream_output, &app_state.app_config, user, Some(server_info.get_base_url().as_str())); - return axum::Json(pli.to_document(&options)).into_response(); + return axum::Json(pli.to_info_document(&options)).into_response(); } if pli.provider_id > 0 { @@ -1035,6 +861,7 @@ async fn xtream_get_stream_info_response( ) .await { + return try_unwrap_body!(axum::response::Response::builder() .status(axum::http::StatusCode::OK) .header( @@ -1236,7 +1063,7 @@ async fn xtream_player_api_handle_content_action( ); } return Some( - serve_file(&file_path, mime::APPLICATION_JSON) + serve_file(&file_path, mime::APPLICATION_JSON.to_string()) .await .into_response(), ); @@ -1302,10 +1129,10 @@ async fn xtream_get_catchup_response( .and_then(|id| id.parse::().ok()) { let uuid = generate_playlist_uuid( - &hex_encode(&pli.get_uuid()), + &pli.get_uuid().to_string(), &catchup_provider_id.to_string(), pli.item_type, - &pli.url, + &pli.input_name, ); let virtual_id = target_id_mapping.get_and_update_virtual_id( &uuid, diff --git a/backend/src/api/model/playlist_mem_cache.rs b/backend/src/api/model/playlist_mem_cache.rs index 62ff59900..897bb2157 100644 --- a/backend/src/api/model/playlist_mem_cache.rs +++ b/backend/src/api/model/playlist_mem_cache.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; use tokio::sync::RwLock; -use shared::model::{M3uPlaylistItem, XtreamPlaylistItem}; +use shared::model::{M3uPlaylistItem, PlaylistItem, XtreamCluster, XtreamPlaylistItem}; use crate::model::ConfigTarget; use crate::repository::bplustree::BPlusTree; use crate::repository::target_id_mapping::{VirtualIdRecord}; @@ -50,6 +50,38 @@ impl PlaylistStorageState { } } + pub async fn update_playlist_items(&self, target: &ConfigTarget, pli_list: Vec<&XtreamPlaylistItem>) { + if target.use_memory_cache { + if let Some(storage) = self.data.write().await.get_mut(&target.name) { + if let Some(xtream) = storage.xtream.as_mut() { + for pli in pli_list { + match pli.xtream_cluster { + XtreamCluster::Live => &mut xtream.live, + XtreamCluster::Video => &mut xtream.vod, + XtreamCluster::Series => &mut xtream.series, + }.insert(pli.virtual_id, pli.clone()); + } + } + } + } + } + + pub async fn insert_playlist_items(&self, target: &ConfigTarget, pli_list: Vec) { + if target.use_memory_cache { + if let Some(storage) = self.data.write().await.get_mut(&target.name) { + if let Some(xtream) = storage.xtream.as_mut() { + for pli in pli_list { + match pli.header.xtream_cluster { + XtreamCluster::Live => &mut xtream.live, + XtreamCluster::Video => &mut xtream.vod, + XtreamCluster::Series => &mut xtream.series, + }.insert(pli.header.virtual_id, pli.to_xtream()); + } + } + } + } + } + pub async fn cache_playlist(&self, target_name: &str, playlist: PlaylistStorage) { match playlist { PlaylistStorage::M3uPlaylist(m3u_playlist) => { diff --git a/backend/src/library/classifier.rs b/backend/src/library/classifier.rs index a5af44031..841adf6bb 100644 --- a/backend/src/library/classifier.rs +++ b/backend/src/library/classifier.rs @@ -1,68 +1,24 @@ -use log::debug; -use regex::Regex; -use std::path::Path; -use shared::utils::CONSTANTS; -use crate::model::LibraryConfig; use crate::library::scanner::ScannedMediaFile; -use crate::library::{SeriesEpisodeFile, SeriesKey}; +use crate::library::{SeriesKey}; +use crate::ptt::{ptt_parse_title, PttMetadata}; -fn clear_filename(file_name: &str) -> String { - // Remove quality indicators (1080p, 720p, BluRay, etc.) - let mut cleaned = CONSTANTS.re_quality.replace(file_name, "").to_string(); - - // Clean up special characters - cleaned = cleaned.replace(['.', '_', '-'], " ").trim().to_string(); - - // remove brackets - cleaned = CONSTANTS.re_classifier_brackets_info.replace_all(&cleaned, "").to_string(); - - // put space between camel case - cleaned = CONSTANTS.re_classifier_camel_case.replace_all(&cleaned, "$1 $2").to_string(); - - // Remove multiple spaces - CONSTANTS.re_whitespace.replace_all(&cleaned, " ").trim().to_string() -} - -#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq)] -pub enum MovieDbId { - Tmdb(u32), - Tvdb(u32), -} - -impl MovieDbId { - fn get_id(ids: Option<&Vec>, f: F) -> Option - where - F: Fn(&MovieDbId) -> Option, - { - ids.as_ref()?.iter().find_map(f) - } - - pub fn get_tmdb_id(ids: Option<&Vec>) -> Option { - Self::get_id(ids, |id| { - if let MovieDbId::Tmdb(val) = id { Some(*val) } else { None } - }) - } - - pub fn get_tvdb_id(ids: Option<&Vec>) -> Option { - Self::get_id(ids, |id| { - if let MovieDbId::Tvdb(val) = id { Some(*val) } else { None } - }) - } -} - -/// Classification result for a video file +// Classification result for a video file #[derive(Debug, Clone, PartialEq, Eq)] pub enum MediaClassification { - Movie, + Movie { + metadata: PttMetadata + }, Series { - season: Option, - episode: Option, + key: SeriesKey, + episode: u32, + season: u32, + metadata: PttMetadata, }, } impl MediaClassification { pub fn is_movie(&self) -> bool { - matches!(self, MediaClassification::Movie) + matches!(self, MediaClassification::Movie { .. }) } pub fn is_series(&self) -> bool { @@ -72,171 +28,29 @@ impl MediaClassification { /// Classifier for determining if a video file is a movie or series pub struct MediaClassifier { - series_patterns: Vec, } impl MediaClassifier { - /// Creates a new classifier from the VOD configuration - pub fn from_config(config: &LibraryConfig) -> Self { - Self { - series_patterns: config.classification.series_patterns.clone(), - } - } - // Classifies a video file as either Movie or Series - pub fn classify(&self, file: &ScannedMediaFile) -> MediaClassification { + pub fn classify(file: &ScannedMediaFile) -> MediaClassification { let file_name = &file.file_name; - let parent_path = file.path.parent().and_then(|p| p.to_str()).unwrap_or(""); - // Check if any series pattern matches - for pattern in &self.series_patterns { - if pattern.is_match(file_name) || pattern.is_match(parent_path) { - debug!("File '{file_name}' matched series pattern: {pattern}"); - return Self::extract_series_info(file_name, parent_path); + let ptt_metadata = ptt_parse_title(file_name); + + if let (Some(episode), Some(season)) = (ptt_metadata.episodes.first(), ptt_metadata.seasons.first()) { + MediaClassification::Series { + key: SeriesKey { + title: ptt_metadata.title.clone(), + year: ptt_metadata.year, + tmdb_id: ptt_metadata.tmdb, + }, + episode: *episode, + season: *season, + metadata: ptt_metadata, } + } else { + MediaClassification::Movie {metadata: ptt_metadata } } - - // If no series pattern matches, classify as movie - debug!("File '{file_name}' classified as Movie (no series pattern match)"); - MediaClassification::Movie - } - - pub fn classify_series_file(file: &ScannedMediaFile) -> Option<(SeriesKey, u32, u32)> { - let file_name = Path::new(&file.file_name) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or(&file.file_name); - - let (ids, show_name) = Self::extract_show_name(file); - - let parent_path = file.path.parent() - .and_then(|p| p.to_str()) - .unwrap_or(""); - - let classification = Self::extract_series_info(file_name, parent_path); - - if let MediaClassification::Series { season, episode } = classification { - if let (Some(season), Some(episode)) = (season, episode) { - let tmdb_id = MovieDbId::get_tmdb_id(ids.as_ref()); - let year = None; // TODO extract from somewhere - let key = SeriesKey { - title: show_name, - tmdb_id, - year, - }; - return Some((key, season, episode)); - } - } - - None - } - - /// Extracts season and episode information from file name or path - fn extract_series_info(file_name: &str, parent_path: &str) -> MediaClassification { - let combined = format!("{parent_path} {file_name}"); - - // Try to extract season and episode numbers - // Common patterns: S01E02, s01e02, 1x02, Season 1 Episode 2, etc. - let season = Self::extract_season(&combined); - let episode = Self::extract_episode(&combined); - - if season.is_some() || episode.is_some() { - debug!("Extracted series info - Season: {season:?}, Episode: {episode:?}"); - } - - MediaClassification::Series { season, episode } - } - - /// Extracts season number from text using common patterns - fn extract_season(text: &str) -> Option { - // Patterns: S01, s01, Season 1, season 1, Season1, etc. - CONSTANTS.re_classifier_season - .captures(text) - .and_then(|cap| cap.get(1)) - .and_then(|m| m.as_str().parse::().ok()) - } - - /// Extracts episode number from text using common patterns - fn extract_episode(text: &str) -> Option { - // Patterns: E02, e02, Episode 2, episode 2, x02, etc. - CONSTANTS.re_classifier_episode - .captures(text) - .and_then(|cap| cap.get(1)) - .and_then(|m| m.as_str().parse::().ok()) - } - - /// Extracts show name from file path for series - /// Removes season/episode patterns and cleans up the name - pub fn extract_show_name(file: &ScannedMediaFile) -> (Option>, String) { - let file_name = Path::new(&file.file_name) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or(&file.file_name); - - let moviedb_ids: Vec = CONSTANTS.re_classifier_moviedb_id.captures_iter(file_name) - .filter_map(|caps| { - let id = caps[2].parse::().ok()?; - match &caps[1].to_lowercase()[..] { - "tmdb" => Some(MovieDbId::Tmdb(id)), - "tvdb" => Some(MovieDbId::Tvdb(id)), - _ => None, - } - }) - .collect(); - - - // Remove common series patterns - let cleaned = CONSTANTS.re_classifier_cleanup.replace(file_name, "").trim().to_string(); - // Clean up remaining special characters - ((!moviedb_ids.is_empty()).then_some(moviedb_ids), clear_filename(&cleaned)) - } - - /// Extracts movie title from file path - /// Attempts to extract tmdb-id, tvdb-id, year and remove quality tags - pub fn extract_movie_search_info(file: &ScannedMediaFile) -> (Option>, String, Option) { - let file_name = Path::new(&file.file_name) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or(&file.file_name); - - let moviedb_ids: Vec = CONSTANTS.re_classifier_moviedb_id.captures_iter(file_name) - .filter_map(|caps| { - let id = caps[2].parse::().ok()?; - match &caps[1].to_lowercase()[..] { - "tmdb" => Some(MovieDbId::Tmdb(id)), - "tvdb" => Some(MovieDbId::Tvdb(id)), - _ => None, - } - }) - .collect(); - - // Try to extract year (4 digits in parentheses or standalone) - let year = CONSTANTS.re_classifier_year - .captures(file_name) - .and_then(|cap| cap.get(1)) - .and_then(|m| m.as_str().parse::().ok()) - .filter(|&y| (1900..=2100).contains(&y)); // Validate year range - - // Remove year and everything after it, and quality tags - let mut title = file_name.to_string(); - if let Some(mat) = CONSTANTS.re_classifier_year.find(file_name) { - title = file_name[..mat.start()].to_string(); - } - - title = clear_filename(&title); - - ((!moviedb_ids.is_empty()).then_some(moviedb_ids), title, year) - } - - pub fn build_series_key(episodes: &[SeriesEpisodeFile]) -> SeriesKey { - if let Some(first) = episodes.first() { - if let Some((key, _season, _episode)) = Self::classify_series_file(&first.file) { - return key; - } - } - - // fallback, not really helpful - SeriesKey {title: "Unknown".to_string(), tmdb_id: None, year: None} } } @@ -256,72 +70,51 @@ mod tests { } } - #[test] - fn test_season_extraction() { - assert_eq!(MediaClassifier::extract_season("S01E02"), Some(1)); - assert_eq!(MediaClassifier::extract_season("s03e05"), Some(3)); - assert_eq!(MediaClassifier::extract_season("Season 2 Episode 1"), Some(2)); - assert_eq!(MediaClassifier::extract_season("season_05_episode_03"), Some(5)); - } - - #[test] - fn test_episode_extraction() { - assert_eq!(MediaClassifier::extract_episode("S01E02"), Some(2)); - assert_eq!(MediaClassifier::extract_episode("s03e05"), Some(5)); - assert_eq!(MediaClassifier::extract_episode("Episode 12"), Some(12)); - assert_eq!(MediaClassifier::extract_episode("1x15"), Some(15)); - } - #[test] fn test_extract_show_name() { let file = create_test_file("Breaking.Bad.S01E01.mkv", "/tv/Breaking.Bad"); - let (_, show_name) = MediaClassifier::extract_show_name(&file); - assert_eq!(show_name, "Breaking Bad"); + let classification = MediaClassifier::classify(&file); + match classification { + MediaClassification::Movie { .. } => { + + } + MediaClassification::Series { key, episode, season, metadata, .. } => { + assert_eq!(key.title, "Breaking Bad"); + assert_eq!(episode, 1); + assert_eq!(season, 1); + assert_eq!(metadata.extension, Some("mkv".to_string())); + } + } } #[test] fn test_extract_movie_title() { let file = create_test_file("The.Matrix.1999.1080p.BluRay.mkv", "/movies"); - let (_moviedb_id, title, year) = MediaClassifier::extract_movie_search_info(&file); - assert_eq!(title, "The Matrix"); - assert_eq!(year, Some(1999)); + let classification = MediaClassifier::classify(&file); + match classification { + MediaClassification::Movie { metadata } => { + assert_eq!(metadata.title, "The Matrix"); + assert_eq!(metadata.year, Some(1999)); + assert_eq!(metadata.extension, Some("mkv".to_string())); + + } + MediaClassification::Series { .. } => { + } + } } #[test] fn test_extract_movie_title_without_year() { let file = create_test_file("Inception.1080p.BluRay.mkv", "/movies"); - let (tmdbid, title, year) = MediaClassifier::extract_movie_search_info(&file); - assert_eq!(tmdbid, None); - assert_eq!(title, "Inception"); - assert_eq!(year, None); - } - - #[test] - fn test_classify_series() { - let series_pattern = Regex::new(r"(?i)s\d+e\d+").unwrap(); - let classifier = MediaClassifier { - series_patterns: vec![series_pattern], - }; - - let file = create_test_file("Breaking.Bad.S01E01.mkv", "/tv"); - let classification = classifier.classify(&file); - assert!(classification.is_series()); - - if let MediaClassification::Series { season, episode } = classification { - assert_eq!(season, Some(1)); - assert_eq!(episode, Some(1)); + let classification = MediaClassifier::classify(&file); + match classification { + MediaClassification::Movie { metadata } => { + assert_eq!(metadata.title, "Inception"); + assert_eq!(metadata.year, None); + assert_eq!(metadata.extension, Some("mkv".to_string())); + } + MediaClassification::Series { .. } => { + } } } - - #[test] - fn test_classify_movie() { - let series_pattern = Regex::new(r"(?i)s\d+e\d+").unwrap(); - let classifier = MediaClassifier { - series_patterns: vec![series_pattern], - }; - - let file = create_test_file("The.Matrix.1999.mkv", "/movies"); - let classification = classifier.classify(&file); - assert!(classification.is_movie()); - } } diff --git a/backend/src/library/metadata.rs b/backend/src/library/metadata.rs index 49b313066..6cce3f9e4 100644 --- a/backend/src/library/metadata.rs +++ b/backend/src/library/metadata.rs @@ -34,42 +34,42 @@ pub struct VideoClipMetadata { pub struct MovieMetadata { pub title: String, // Original title (if different from title) - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub original_title: Option, // Release year - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub year: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub plot: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub tagline: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub runtime: Option, // MPAA rating (e.g., "PG-13", "R") - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub mpaa: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub imdb_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub tmdb_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub tvdb_id: Option, // Rating (0.0 - 10.0) - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rating: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub genres: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub directors: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub writers: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub actors: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub studios: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub poster: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub fanart: Option, pub source: MetadataSource, pub last_updated: i64, @@ -81,44 +81,44 @@ pub struct MovieMetadata { pub struct SeriesMetadata { pub title: String, // Original title (if different) - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub original_title: Option, // First aired year - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub year: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub plot: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub mpaa: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub imdb_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub tmdb_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub tvdb_id: Option, // Rating (0.0 - 10.0) - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rating: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub genres: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub directors: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub writers: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub actors: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub studios: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub poster: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub fanart: Option, // Status (e.g., "Continuing", "Ended") - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub status: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub seasons: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub episodes: Option>, #[serde(default)] pub source: MetadataSource, @@ -147,16 +147,16 @@ pub struct EpisodeMetadata { #[serde(default)] pub episode: u32, // Aired date (ISO 8601 format) - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub aired: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub plot: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub runtime: Option, // Rating (0.0 - 10.0) - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rating: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub thumb: Option, pub file_path: String, pub file_size: u64, @@ -187,9 +187,9 @@ pub struct SeasonMetadata { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Actor { pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub role: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub thumb: Option, } @@ -234,8 +234,8 @@ impl MediaMetadata { pub fn poster(&self) -> Option<&str> { match self { - MediaMetadata::Movie(m) => m.poster.as_deref(), - MediaMetadata::Series(s) => s.poster.as_deref(), + MediaMetadata::Movie(m) => m.poster.as_deref().or(m.fanart.as_deref()), + MediaMetadata::Series(s) => s.poster.as_deref().or(s.fanart.as_deref()), } } diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index 54cefe491..d6651f525 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -1,12 +1,11 @@ use log::{debug, error, info, warn}; -use crate::library::classifier::{MediaClassification, MediaClassifier}; use crate::library::metadata::{MediaMetadata, MetadataSource, MovieMetadata, SeriesMetadata}; -use crate::library::nfo_reader::NfoReader; use crate::library::scanner::ScannedMediaFile; use crate::library::tmdb_client::TmdbClient; -use crate::library::{MediaGroup, MetadataStorage, MovieDbId, TMDB_API_KEY}; +use crate::library::{MediaGroup, MetadataStorage, TMDB_API_KEY}; use crate::model::LibraryConfig; +use crate::ptt::{ptt_parse_title, PttMetadata}; // Metadata resolver that tries multiple sources to get video metadata pub struct MetadataResolver { @@ -35,17 +34,17 @@ impl MetadataResolver { debug!("Resolving metadata for: {group}"); // Step 1: Classify the file - let (Some(file), classification) = (match group { - MediaGroup::Movie { file } => (Some(file), MediaClassification::Movie), + let (movie, Some(file), metadata) = (match group { + MediaGroup::Movie { file, metadata } => (true, Some(file), metadata), MediaGroup::Series { show_key: _, episodes } => { let episode = episodes.first()?; - (Some(&episode.file), MediaClassification::Series { season: None, episode: None }) - }, + (false, Some(&episode.file), &episode.metadata) + } }) else { return None }; // Step 2: Try TMDB if enabled if let Some(ref tmdb) = self.tmdb_client { - match self.resolve_from_tmdb(file, &classification, tmdb).await { + match self.resolve_from_tmdb(movie, file, metadata, tmdb).await { Ok(Some(metadata)) => { info!("Found TMDB metadata for: {}", file.file_path); return Some(metadata); @@ -58,18 +57,18 @@ impl MetadataResolver { } // TODO series implementation missing - if classification == MediaClassification::Movie { - // Step 3: Try to read existing NFO file - if let Some(metadata) = NfoReader::read_metadata(&file.path).await { - info!("Found NFO metadata for: {}", file.file_path); - return Some(metadata); - } - } + // if classification == MediaClassification::Movie { + // // Step 3: Try to read existing NFO file + // if let Some(metadata) = NfoReader::read_metadata(&file.path).await { + // info!("Found NFO metadata for: {}", file.file_path); + // return Some(metadata); + // } + // } // Step 4: Fallback to filename parsing if self.fallback_to_filename { info!("Using filename-based metadata for: {}", file.file_path); - Some(Self::resolve_from_filename(file, &classification)) + Some(Self::resolve_from_filename(movie, metadata)) } else { warn!("No metadata found for: {}", file.file_path); None @@ -77,66 +76,49 @@ impl MetadataResolver { } // Attempts to resolve metadata from TMDB - async fn resolve_from_tmdb(&self, file: &ScannedMediaFile, classification: &MediaClassification, tmdb: &TmdbClient) -> Result, String> { - match classification { - MediaClassification::Movie => { - let (moviedb_ids, title, year) = MediaClassifier::extract_movie_search_info(file); - let tmdb_id = MovieDbId::get_tmdb_id(moviedb_ids.as_ref()); - tmdb.search_movie(tmdb_id, &title, year).await - } - MediaClassification::Series { .. } => { - let (seriesdb_id, show_name) = MediaClassifier::extract_show_name(file); - let tmdb_id = MovieDbId::get_tmdb_id(seriesdb_id.as_ref()); - debug!("Searching TMDB for series: {show_name}"); + async fn resolve_from_tmdb(&self, movie: bool, file: &ScannedMediaFile, metadata: &PttMetadata, tmdb: &TmdbClient) -> Result, String> { + if movie { + let tmdb_id = metadata.tmdb.as_ref(); + tmdb.search_movie(tmdb_id, metadata.title.as_str(), metadata.year.as_ref()).await + } else { + let (series_year, tmdb_id) = if metadata.year.is_some() { (metadata.year, metadata.tmdb) } else { // Try to extract year from parent directory if available - let year = file.path.parent() + file.path.parent() .and_then(|p| p.file_name()) .and_then(|n| n.to_str()) - .and_then(|s| { - // Look for 4-digit year in directory name - s.chars() - .collect::>() - .windows(4) - .find_map(|w| { - let year_str: String = w.iter().collect(); - year_str.parse::().ok() - .filter(|&y| (1900..=2100).contains(&y)) - }) - }); - tmdb.search_series(tmdb_id, &show_name, year).await - } + .map_or((None, None), |s| { + let ptt = ptt_parse_title(s); + (ptt.year, ptt.tmdb) + }) + }; + tmdb.search_series(tmdb_id, metadata.title.as_str(), series_year).await } } // Creates basic metadata from filename parsing - fn resolve_from_filename( - file: &ScannedMediaFile, - classification: &MediaClassification, - ) -> MediaMetadata { + fn resolve_from_filename(movie: bool, metadata: &PttMetadata) -> MediaMetadata { let timestamp = chrono::Utc::now().timestamp(); - match classification { - MediaClassification::Movie => { - let (moviedb_id, title, year) = MediaClassifier::extract_movie_search_info(file); - MediaMetadata::Movie(MovieMetadata { - title, - year, - tmdb_id: MovieDbId::get_tmdb_id(moviedb_id.as_ref()), - tvdb_id: MovieDbId::get_tvdb_id(moviedb_id.as_ref()), - source: MetadataSource::FilenameParsed, - last_updated: timestamp, - ..MovieMetadata::default() - }) - } - MediaClassification::Series { season: _, episode: _ } => { - let (_moviedb_id, show_name) = MediaClassifier::extract_show_name(file); - MediaMetadata::Series(SeriesMetadata { - title: show_name, - source: MetadataSource::FilenameParsed, - last_updated: timestamp, - ..SeriesMetadata::default() - }) - } + if movie { + MediaMetadata::Movie(MovieMetadata { + title: metadata.title.clone(), + year: metadata.year, + tmdb_id: metadata.tmdb, + tvdb_id: metadata.tvdb, + source: MetadataSource::FilenameParsed, + last_updated: timestamp, + ..MovieMetadata::default() + }) + } else { + MediaMetadata::Series(SeriesMetadata { + title: metadata.title.clone(), + year: metadata.year, + tmdb_id: metadata.tmdb, + tvdb_id: metadata.tvdb, + source: MetadataSource::FilenameParsed, + last_updated: timestamp, + ..SeriesMetadata::default() + }) } } } @@ -144,9 +126,10 @@ impl MetadataResolver { #[cfg(test)] mod tests { use super::*; - use crate::model::{LibraryClassificationConfig, LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; + use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; use std::path::PathBuf; use std::time::Duration; + use crate::library::{MediaClassification, MediaClassifier}; fn create_test_config(tmdb_enabled: bool) -> LibraryConfig { LibraryConfig { @@ -174,10 +157,6 @@ mod tests { fallback_to_filename: true, formats: vec![], }, - classification: LibraryClassificationConfig { - series_patterns: vec![], - series_directory_patterns: vec![], - }, playlist: LibraryPlaylistConfig { movie_category: "Movies".to_string(), series_category: "Series".to_string(), @@ -206,7 +185,11 @@ mod tests { .unwrap_or_else(|_| reqwest::Client::new()); let resolver = MetadataResolver::from_config(&config, client, MetadataStorage::new(PathBuf::from("/tmp"))); let file = create_test_file("The.Matrix.1999.1080p.mkv"); - let group = MediaGroup::Movie { file }; + let metadata = match MediaClassifier::classify(&file) { + MediaClassification::Movie { metadata, .. } => metadata, + MediaClassification::Series { metadata, .. } => metadata, + }; + let group = MediaGroup::Movie { file, metadata }; let metadata = resolver.resolve(&group).await; assert!(metadata.is_some()); @@ -214,7 +197,7 @@ mod tests { if let Some(MediaMetadata::Movie(movie)) = metadata { assert_eq!(movie.title, "The Matrix"); assert_eq!(movie.year, Some(1999)); - assert_eq!(movie.source, MetadataSource::FilenameParsed); + assert_eq!(movie.source, MetadataSource::Tmdb); } else { panic!("Expected movie metadata"); } @@ -231,7 +214,11 @@ mod tests { config.metadata.fallback_to_filename = false; let resolver = MetadataResolver::from_config(&config, client, MetadataStorage::new(PathBuf::from("/tmp"))); let file = create_test_file("Unknown.Movie.mkv"); - let group = MediaGroup::Movie { file }; + let metadata = match MediaClassifier::classify(&file) { + MediaClassification::Movie { metadata, .. } => metadata, + MediaClassification::Series { metadata, .. } => metadata, + }; + let group = MediaGroup::Movie { file, metadata }; let metadata = resolver.resolve(&group).await; assert!(metadata.is_none()); diff --git a/backend/src/library/processor.rs b/backend/src/library/processor.rs index 15e6b545c..7691f8abd 100644 --- a/backend/src/library/processor.rs +++ b/backend/src/library/processor.rs @@ -96,7 +96,7 @@ impl LibraryProcessor { let scanned_paths: std::collections::HashSet<_> = media_groups .iter() .flat_map(|group| match group { - MediaGroup::Movie { file } => vec![file.file_path.as_str()], + MediaGroup::Movie { file, .. } => vec![file.file_path.as_str()], MediaGroup::Series { episodes, .. } => episodes.iter().map(|ep| ep.file.file_path.as_str()).collect(), }) .collect(); @@ -118,7 +118,7 @@ impl LibraryProcessor { async fn process_group(&self, group: &MediaGroup, existing_map: &HashMap, force_rescan: bool) -> Result { match group { - MediaGroup::Movie { file: _ } => { + MediaGroup::Movie { file: _, .. } => { self.process_movie(group, existing_map, force_rescan).await } MediaGroup::Series { show_key: _, episodes: _ } => { @@ -130,7 +130,7 @@ impl LibraryProcessor { // Processes a single video file async fn process_movie(&self, group: &MediaGroup, existing_map: &HashMap, force_rescan: bool, ) -> Result { - let MediaGroup::Movie { file } = group else { return Err(format!("Expected movie to resolve but got {group}")) }; + let MediaGroup::Movie { file, .. } = group else { return Err(format!("Expected movie to resolve but got {group}")) }; // Check if file already exists in cache let (cache_entry, status) = if let Some(existing_entry) = existing_map.get(&file.file_path) { // Check if file has been modified diff --git a/backend/src/library/scanner.rs b/backend/src/library/scanner.rs index d8c7c70ce..cd4ebb238 100644 --- a/backend/src/library/scanner.rs +++ b/backend/src/library/scanner.rs @@ -1,4 +1,4 @@ -use crate::library::MediaClassifier; +use crate::library::{MediaClassification, MediaClassifier}; use crate::model::{LibraryConfig, LibraryScanDirectory}; use log::{debug, error, info, trace, warn}; use std::collections::HashMap; @@ -6,6 +6,7 @@ use std::fmt::{Display, Formatter}; use std::path::{Path, PathBuf}; use tokio::fs; use tokio::io; +use crate::ptt::PttMetadata; #[derive(Hash, Eq, PartialEq, Clone, Debug)] pub struct SeriesKey { @@ -30,6 +31,7 @@ impl Display for SeriesKey { pub enum MediaGroup { Movie { file: ScannedMediaFile, + metadata: Box, }, Series { show_key: SeriesKey, @@ -40,7 +42,7 @@ pub enum MediaGroup { impl Display for MediaGroup { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - MediaGroup::Movie { file } => write!(f, "{}", file.file_path), + MediaGroup::Movie { file, .. } => write!(f, "{}", file.file_path), MediaGroup::Series { show_key, .. } => write!(f, "{}", show_key.title), } } @@ -51,6 +53,7 @@ pub struct SeriesEpisodeFile { pub file: ScannedMediaFile, pub season: u32, pub episode: u32, + pub metadata: Box, } pub struct MediaGrouper; @@ -61,17 +64,22 @@ impl MediaGrouper { let mut movies = Vec::new(); for file in files { - if let Some((key, season, episode)) = MediaClassifier::classify_series_file(&file) { - series_map - .entry(key) - .or_default() - .push(SeriesEpisodeFile { - file, - season, - episode, - }); - } else { - movies.push(MediaGroup::Movie { file }); + let classification = MediaClassifier::classify(&file); + match classification { + MediaClassification::Movie { metadata } => { + movies.push(MediaGroup::Movie { file, metadata: Box::new(metadata) }); + } + MediaClassification::Series { key, metadata, season, episode,.. } => { + series_map + .entry(key) + .or_default() + .push(SeriesEpisodeFile { + file, + season, + episode, + metadata: Box::new(metadata), + }); + } } } @@ -274,7 +282,7 @@ impl LibraryScanner { #[cfg(test)] mod tests { use super::*; - use crate::model::{LibraryClassificationConfig, LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; + use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; fn create_test_config() -> LibraryConfig { LibraryConfig { @@ -302,10 +310,6 @@ mod tests { fallback_to_filename: true, formats: vec![], }, - classification: LibraryClassificationConfig { - series_patterns: vec![], - series_directory_patterns: vec![], - }, playlist: LibraryPlaylistConfig { movie_category: "Local Movies".to_string(), series_category: "Local Series".to_string(), diff --git a/backend/src/library/tmdb_client.rs b/backend/src/library/tmdb_client.rs index 3d63ce8e8..5cc264dd5 100644 --- a/backend/src/library/tmdb_client.rs +++ b/backend/src/library/tmdb_client.rs @@ -38,11 +38,11 @@ impl TmdbClient { } // Searches for a movie by title and optional year - pub async fn search_movie(&self, tmdb_id: Option, title: &str, year: Option) -> Result, String> { + pub async fn search_movie(&self, tmdb_id: Option<&u32>, title: &str, year: Option<&u32>) -> Result, String> { debug!("TMDB search movie: {title}"); if let Some(movie_id) = tmdb_id { - return self.fetch_movie_details(movie_id).await; + return self.fetch_movie_details(*movie_id).await; } sleep(Duration::from_millis(self.rate_limit_ms)).await; @@ -62,7 +62,7 @@ impl TmdbClient { } } - fn build_movie_search_url(&self, title: &str, year: Option) -> Result { + fn build_movie_search_url(&self, title: &str, year: Option<&u32>) -> Result { let mut url = Url::parse(&format!("{TMDB_API_BASE_URL}/search/movie")).map_err(|e| format!("Failed to parse URL for TMDB movie search: {e}"))?; { let mut q = url.query_pairs_mut(); @@ -100,6 +100,8 @@ impl TmdbClient { // Searches for a TV series by title and optional year pub async fn search_series(&self, tmdb_id: Option, title: &str, year: Option) -> Result, String> { + debug!("Searching TMDB for series: {title}"); + let key = format!("{title}-{tmdb_id:?}-{year:?}"); if self.fetched_series_key.read().await.contains(&key) { diff --git a/backend/src/main.rs b/backend/src/main.rs index d2ddaf924..170c036c7 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -32,7 +32,7 @@ use std::sync::Arc; #[command(name = "tuliprox")] #[command(author = "euzu ")] #[command(version)] -#[command(about = "Extended M3U playlist filter", long_about = None)] +#[command(about = "Extended playlist proxy", long_about = None)] struct Args { /// The config directory #[arg(short = 'p', long = "config-path")] diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index c18561f77..395496358 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -1,6 +1,4 @@ use crate::model::macros; -use log::error; -use regex::Regex; use shared::error::TuliproxError; use shared::model::{default_metadata_path, LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; use std::path::PathBuf; @@ -39,12 +37,6 @@ pub struct LibraryTmdbConfig { pub language: String, } -#[derive(Debug, Clone)] -pub struct LibraryClassificationConfig { - pub series_patterns: Vec, - pub series_directory_patterns: Vec, -} - #[derive(Debug, Clone)] pub struct LibraryPlaylistConfig { pub movie_category: String, @@ -58,7 +50,6 @@ pub struct LibraryConfig { pub scan_directories: Vec, pub supported_extensions: Vec, pub metadata: LibraryMetadataConfig, - pub classification: LibraryClassificationConfig, pub playlist: LibraryPlaylistConfig, } @@ -93,37 +84,6 @@ macros::from_impl!(LibraryConfig); impl From<&LibraryConfigDto> for LibraryConfig { fn from(dto: &LibraryConfigDto) -> Self { - // Compile series patterns - let series_patterns = dto - .classification - .series_patterns - .iter() - .filter_map(|pattern| { - match Regex::new(pattern) { - Ok(re) => Some(re), - Err(e) => { - error!("Invalid series pattern '{pattern}': {e}"); - None - } - } - }) - .collect(); - - // Compile directory patterns - let series_directory_patterns = dto - .classification - .series_directory_patterns - .iter() - .filter_map(|pattern| { - match Regex::new(pattern) { - Ok(re) => Some(re), - Err(e) => { - error!("Invalid series directory pattern '{pattern}': {e}"); - None - } - } - }) - .collect(); Self { enabled: dto.enabled, @@ -159,10 +119,6 @@ impl From<&LibraryConfigDto> for LibraryConfig { fallback_to_filename: dto.metadata.fallback_to_filename, formats: dto.metadata.formats.clone(), }, - classification: LibraryClassificationConfig { - series_patterns, - series_directory_patterns, - }, playlist: LibraryPlaylistConfig { movie_category: dto.playlist.movie_category.clone(), series_category: dto.playlist.series_category.clone(), diff --git a/backend/src/model/config/log.rs b/backend/src/model/config/log.rs index 02a18eea0..ea1493fbb 100644 --- a/backend/src/model/config/log.rs +++ b/backend/src/model/config/log.rs @@ -14,13 +14,13 @@ pub struct LogConfig { pub sanitize_sensitive_info: bool, #[serde(default)] pub log_active_user: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub log_level: Option, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] pub struct LogLevelConfig { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub log: Option, } diff --git a/backend/src/model/config/target.rs b/backend/src/model/config/target.rs index 0df33e76b..60e63065d 100644 --- a/backend/src/model/config/target.rs +++ b/backend/src/model/config/target.rs @@ -283,6 +283,9 @@ impl ConfigTarget { } pub fn is_force_redirect(&self, item_type: PlaylistItemType) -> bool { + if item_type.is_local() { + return false; + } self.options .as_ref() .and_then(|options| options.force_redirect.as_ref()) diff --git a/backend/src/model/healthcheck.rs b/backend/src/model/healthcheck.rs index ff8de76d5..c4b665bc5 100644 --- a/backend/src/model/healthcheck.rs +++ b/backend/src/model/healthcheck.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; pub struct Healthcheck { pub status: String, pub version: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub build_time: Option, pub server_time: String, } diff --git a/backend/src/model/media_properties.rs b/backend/src/model/media_properties.rs index 2645dc0ea..5d0fe08b4 100644 --- a/backend/src/model/media_properties.rs +++ b/backend/src/model/media_properties.rs @@ -1,5 +1,3 @@ -// backend/src/model/media_properties.rs - use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use std::fmt; @@ -161,24 +159,39 @@ impl MediaQuality { parts.join(separator) } - /// Extracts media quality information from an `ffprobe` info block. - /// The `info_block` is expected to be a `serde_json::Value` object. - pub fn from_ffprobe_info(audio: Option<&String>, video: Option<&String>) -> Option { - let video_info = video.and_then(|v| serde_json::from_str::>(v).ok())?; + fn from_ffprobe_info_audio(audio: Option<&String>) -> Option<(AudioCodec, AudioChannels)> { // Assuming the first audio stream is the primary one. let audio_info = audio.and_then(|v| serde_json::from_str::>(v).ok())?; - // Helper to get a value by trying a prioritized list of field names. - let get_value = |obj: &Map, fields: &[&str]| -> Option { - for field in fields { - if let Some(value) = obj.get(*field) { - if !value.is_null() { - return Some(value.clone()); - } - } - } - None - }; + // 4. Classify audio codec + let audio_codec = get_value(&audio_info, &["codec_name"]) + .and_then(|v| v.as_str().map(str::to_lowercase)) + .map_or(AudioCodec::default(), |name| match name.as_str() { + "aac" => AudioCodec::AAC, + "ac3" => AudioCodec::AC3, + "eac3" => AudioCodec::EAC3, + "dts" => AudioCodec::DTS, + "truehd" => AudioCodec::TrueHD, + "flac" => AudioCodec::FLAC, + _ => AudioCodec::default(), + }); + + // 5. Classify audio channels + let audio_channels = get_value(&audio_info, &["channel_layout"]) + .and_then(|v| v.as_str().map(str::to_lowercase)) + .map_or(AudioChannels::default(), |layout| match layout.as_str() { + l if l.starts_with("7.1") => AudioChannels::Surround71, + l if l.starts_with("5.1") => AudioChannels::Surround51, + "stereo" => AudioChannels::Stereo, + "mono" => AudioChannels::Mono, + _ => AudioChannels::default(), + }); + + Some((audio_codec, audio_channels)) + } + + fn from_ffprobe_info_video(video: Option<&String>) -> Option<(VideoResolution, VideoCodec, VideoDynamicRange)> { + let video_info = video.and_then(|v| serde_json::from_str::>(v).ok())?; // 1. Classify video resolution from width let resolution = get_value(&video_info, &["height", "coded_height"]) @@ -220,36 +233,38 @@ impl MediaQuality { } }; - // 4. Classify audio codec - let audio_codec = get_value(&audio_info, &["codec_name"]) - .and_then(|v| v.as_str().map(str::to_lowercase)) - .map_or(AudioCodec::default(), |name| match name.as_str() { - "aac" => AudioCodec::AAC, - "ac3" => AudioCodec::AC3, - "eac3" => AudioCodec::EAC3, - "dts" => AudioCodec::DTS, - "truehd" => AudioCodec::TrueHD, - "flac" => AudioCodec::FLAC, - _ => AudioCodec::default(), - }); - - // 5. Classify audio channels - let audio_channels = get_value(&audio_info, &["channel_layout"]) - .and_then(|v| v.as_str().map(str::to_lowercase)) - .map_or(AudioChannels::default(), |layout| match layout.as_str() { - l if l.starts_with("7.1") => AudioChannels::Surround71, - l if l.starts_with("5.1") => AudioChannels::Surround51, - "stereo" => AudioChannels::Stereo, - "mono" => AudioChannels::Mono, - _ => AudioChannels::default(), - }); - - Some(Self { - resolution, - video_codec, - dynamic_range, - audio_codec, - audio_channels, - }) + Some((resolution, video_codec, dynamic_range)) } -} \ No newline at end of file + + /// Extracts media quality information from an `ffprobe` info block. + /// The `info_block` is expected to be a `serde_json::Value` object. + pub fn from_ffprobe_info(audio: Option<&String>, video: Option<&String>) -> Option { + match (Self::from_ffprobe_info_video(video), Self::from_ffprobe_info_audio(audio)) { + (Some((resolution, video_codec, dynamic_range)), + Some((audio_codec, audio_channels))) => { + Some(Self { + resolution, + video_codec, + dynamic_range, + audio_codec, + audio_channels, + }) + } + // TODO to restrictive, maybe only video or audio could be processed + _ => None, + } + } +} + + +// Helper to get a value by trying a prioritized list of field names. +fn get_value(obj: &Map, fields: &[&str]) -> Option { + for field in fields { + if let Some(value) = obj.get(*field) { + if !value.is_null() { + return Some(value.clone()); + } + } + } + None +} diff --git a/backend/src/model/playlist_categories.rs b/backend/src/model/playlist_categories.rs index a147598e0..006e16055 100644 --- a/backend/src/model/playlist_categories.rs +++ b/backend/src/model/playlist_categories.rs @@ -1,9 +1,9 @@ #[derive(Debug, serde::Serialize, serde::Deserialize, Default)] pub struct PlaylistCategories { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub live: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub vod: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub series: Option>, } diff --git a/backend/src/model/xtream.rs b/backend/src/model/xtream.rs index e96a97dfb..58b38522b 100644 --- a/backend/src/model/xtream.rs +++ b/backend/src/model/xtream.rs @@ -2,7 +2,7 @@ use crate::model::{AppConfig, ProxyUserCredentials}; use crate::model::{ConfigTarget, XtreamTargetOutput}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use shared::model::{xtream_const, PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, ProxyUserStatus, XtreamMappingOptions}; +use shared::model::{PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, ProxyUserStatus, XtreamMappingOptions}; use shared::utils::{deserialize_as_string, deserialize_number_from_string_or_zero}; use enum_iterator::all; @@ -76,43 +76,6 @@ pub fn normalize_release_date(document: &mut serde_json::Map) { } } -fn make_bdpath_resource_url(resource_url: &str, bd_path: &str, index: usize, field_prefix: &str) -> String { - if bd_path.starts_with("http") { - format!("{resource_url}/{field_prefix}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH) - } else { - bd_path.to_string() - } -} - -pub fn rewrite_doc_urls(resource_url: Option<&String>, document: &mut serde_json::Map, fields: &[&str], field_prefix: &str) { - if let Some(rewrite_url) = resource_url { - if let Some(bdpath) = document.get(xtream_const::XC_PROP_BACKDROP_PATH) { - match bdpath { - Value::String(bd_path) => { - document.insert(xtream_const::XC_PROP_BACKDROP_PATH.to_string(), Value::Array(vec![Value::String(make_bdpath_resource_url(rewrite_url.as_str(), bd_path, 0, field_prefix))])); - } - Value::Array(bd_path) => { - document.insert(xtream_const::XC_PROP_BACKDROP_PATH.to_string(), Value::Array( - bd_path.iter() - .filter_map(|val| val.as_str()) - .enumerate() - .map(|(index, value)| Value::String(make_bdpath_resource_url(rewrite_url.as_str(), value, index, field_prefix))) - .collect())); - } - _ => {} - } - } - for &field in fields { - if let Some(Value::String(value)) = document.get(field) { - if value.starts_with("http") { - document.insert(field.to_string(), Value::String(format!("{rewrite_url}/{field_prefix}{field}"))); - } - } - } - } -} - - #[derive(Deserialize, Serialize, Clone)] pub struct PlaylistXtreamCategory { #[serde(alias = "category_id")] diff --git a/backend/src/modules.rs b/backend/src/modules.rs index 32d1e15e8..2d13e02b6 100644 --- a/backend/src/modules.rs +++ b/backend/src/modules.rs @@ -13,6 +13,7 @@ macro_rules! include_modules { pub mod utils; pub mod tools; pub mod library; + pub mod ptt; } } diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index ede77df91..8ce42f3f1 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -4,9 +4,7 @@ use crate::model::{XtreamCategory}; use crate::utils::request::DynReader; use crate::utils::xtream::get_xtream_stream_url_base; use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind}; -use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, - PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, - SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster}; +use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, XtreamCluster}; use shared::utils::{generate_playlist_uuid, trim_last_slash}; use std::collections::HashMap; use tokio::task::spawn_blocking; @@ -57,7 +55,7 @@ fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, passwor } } -pub fn parse_xtream_series_info(series_info: &SeriesStreamProperties, group_title: &str, series_name: &str, input: &ConfigInput) -> Option> { +pub fn parse_xtream_series_info(prent_uuid: &UUIDType, series_info: &SeriesStreamProperties, group_title: &str, series_name: &str, input: &ConfigInput) -> Option> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); @@ -65,12 +63,13 @@ pub fn parse_xtream_series_info(series_info: &SeriesStreamProperties, group_titl if let Some(episodes) = series_info.details.as_ref().and_then(|d| d.episodes.as_ref()) { let result: Vec = episodes.iter().map(|episode| { let episode_url = create_xtream_series_episode_url(url, username, password, episode); - let episode_info = EpisodeStreamProperties::from_series(series_info, episode); PlaylistItem { header: PlaylistItemHeader { id: episode.id.to_string(), uuid: generate_playlist_uuid(&input.name, &episode.id.to_string(), PlaylistItemType::Series, &episode_url), + // we use parent_code to track the parent series + parent_code: prent_uuid.to_string(), name: series_name.to_string(), logo: episode.movie_image.clone(), group: group_title.to_string(), @@ -137,9 +136,7 @@ pub async fn parse_xtream(input: &ConfigInput, match map_to_xtream_streams(xtream_cluster, streams).await { Ok(xtream_streams) => { let mut group_map: HashMap = - xtream_categories.into_iter().map(|category| - (category.category_id, category) - ).collect(); + xtream_categories.into_iter().map(|category| (category.category_id, category)).collect(); let mut unknown_grp = XtreamCategory { category_id: 0u32, category_name: "Unknown".to_string(), diff --git a/backend/src/processing/playlist_watch.rs b/backend/src/processing/playlist_watch.rs index a67033c6b..60a8fe1b2 100644 --- a/backend/src/processing/playlist_watch.rs +++ b/backend/src/processing/playlist_watch.rs @@ -5,7 +5,7 @@ use shared::model::{MsgKind, PlaylistGroup}; use crate::messaging::{send_message}; use crate::model::Config; use crate::utils; -use crate::utils::{bincode_deserialize, bincode_serialize}; +use crate::utils::{binary_deserialize, binary_serialize}; pub async fn process_group_watch(client: &reqwest::Client, cfg: &Config, target_name: &str, pl: &PlaylistGroup) { let mut new_tree = BTreeSet::new(); @@ -79,11 +79,11 @@ async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added async fn load_watch_tree(path: &Path) -> Option> { let encoded = tokio::fs::read(path).await.ok()?; - bincode_deserialize(&encoded[..]).ok() + binary_deserialize(&encoded[..]).ok() } async fn save_watch_tree(path: &Path, tree: &BTreeSet) -> std::io::Result<()> { - let encoded: Vec = bincode_serialize(&tree)?; + let encoded: Vec = binary_serialize(&tree)?; tokio::fs::write(path, encoded).await } diff --git a/backend/src/processing/processor/library.rs b/backend/src/processing/processor/library.rs index 3b45c74a5..0fc226cc2 100644 --- a/backend/src/processing/processor/library.rs +++ b/backend/src/processing/processor/library.rs @@ -1,12 +1,12 @@ use crate::library::{MediaMetadata, MetadataAsyncIter, MetadataCacheEntry}; use crate::model::{AppConfig, ConfigInput}; use shared::error::TuliproxError; -use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, StreamProperties, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster}; -use shared::utils::{generate_playlist_uuid, string_to_uuid_type}; +use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster}; +use shared::utils::{generate_playlist_uuid}; use std::path::Path; use std::sync::Arc; -pub async fn get_library_playlist(_client: &reqwest::Client, app_config: &Arc, input: &Arc) -> (Vec, Vec) { +pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &Arc, input: &Arc) -> (Vec, Vec) { let config = &*app_config.config.load(); let Some(library_config) = config.library.as_ref() else { return (vec![], vec![]) }; if !library_config.enabled { return (vec![], vec![]); } @@ -57,10 +57,11 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s let additional_properties = metadata_cache_entry_to_xtream_movie_info(entry); vec![PlaylistItem { header: PlaylistItemHeader { - uuid: string_to_uuid_type(&entry.uuid), + uuid: UUIDType::from_valid_uuid(&entry.uuid), name: metadata.title().to_string(), group: group_name.to_string(), title: metadata.title().to_string(), + logo: metadata.poster().map_or_else(String::new, ToString::to_string), url: format!("file://{}", entry.file_path), xtream_cluster: XtreamCluster::Video, additional_properties, @@ -78,6 +79,7 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s if let Some(details_props) = series_properties.details.as_ref() { if let Some(prop_episodes) = details_props.episodes.as_ref() { for episode in prop_episodes { + let logo = if episode.movie_image.is_empty() { metadata.poster().map_or_else(String::new, ToString::to_string) } else { episode.movie_image.clone() }; let container_extension = Path::new(&episode.direct_source) .extension() .and_then(|s| s.to_str()) @@ -88,6 +90,7 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s // we use parent_code for local series to find the parent series info and straighten the virtual_ids parent_code: entry.uuid.clone(), uuid: generate_playlist_uuid(input_name, &episode.id.to_string(), PlaylistItemType::LocalSeries, &episode.direct_source), + logo: logo.clone(), name: episode.title.clone(), group: group_name.to_string(), title: episode.title.clone(), @@ -103,7 +106,7 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s added: Some(episode.added.clone()), release_date: Some(episode.release_date.clone()), tmdb: episode.tmdb, - movie_image: episode.movie_image.clone(), + movie_image: logo, container_extension, audio: None, video: None, @@ -118,11 +121,12 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s let series_info = PlaylistItem { header: PlaylistItemHeader { - uuid: string_to_uuid_type(&entry.uuid), + uuid: UUIDType::from_valid_uuid(&entry.uuid), id: entry.uuid.clone(), name: metadata.title().to_string(), group: group_name.to_string(), title: metadata.title().to_string(), + logo: metadata.poster().map_or_else(String::new, ToString::to_string), url: format!("file://{}", entry.file_path), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeriesInfo, @@ -167,7 +171,7 @@ pub fn metadata_cache_entry_to_xtream_movie_info( container_extension, rating: movie.rating, rating_5based: None, - stream_type: "movie".to_string(), + stream_type: Some("movie".to_string()), trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())), tmdb: movie.tmdb_id, is_adult: 0, @@ -246,7 +250,7 @@ pub fn metadata_cache_entry_to_xtream_series_info( season: episode.season, title: episode.title.clone(), container_extension, - custom_sid: String::new(), + custom_sid: None, added: episode.file_modified.to_string(), direct_source: episode.file_path.clone(), tmdb: tmdb_id, diff --git a/backend/src/processing/processor/mod.rs b/backend/src/processing/processor/mod.rs index 3f8e54cc3..fc970a4ad 100644 --- a/backend/src/processing/processor/mod.rs +++ b/backend/src/processing/processor/mod.rs @@ -8,28 +8,6 @@ mod sort; pub mod trakt; mod library; -#[macro_export] -macro_rules! handle_error { - ($stmt:expr, $map_err:expr) => { - if let Err(err) = $stmt { - $map_err(err); - } - }; -} -use handle_error; - -#[macro_export] -macro_rules! handle_error_and_return { - ($stmt:expr, $map_err:expr) => { - if let Err(err) = $stmt { - $map_err(err); - return Default::default(); - } - }; -} -use handle_error_and_return; - - // // fn get_resolve__options(target: &ConfigTarget, fpl: &FetchedPlaylist) -> (bool, u16) // diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 99f018429..c6a19641d 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -1,8 +1,7 @@ -use crate::model::{AppConfig, ConfigFavourites, ConfigInput, ConfigRename}; +use crate::model::{AppConfig, Config, ConfigFavourites, ConfigInput, ConfigRename}; use crate::utils::epg; use crate::utils::m3u; use crate::utils::xtream; -use crate::Config; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; @@ -24,7 +23,7 @@ use crate::processing::processor::sort::sort_playlist; use crate::processing::processor::trakt::process_trakt_categories_for_target; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; -use crate::repository::playlist_repository::persist_playlist; +use crate::repository::playlist_repository::{persist_input_playlist, persist_playlist}; use crate::utils::StepMeasure; use crate::utils::{debug_if_enabled, trace_if_enabled}; use deunicode::deunicode; @@ -173,7 +172,7 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType { let mut data = Vec::with_capacity(base_uuid.len() + mapping_id.len()); - data.extend_from_slice(base_uuid); + data.extend_from_slice(base_uuid.as_ref()); data.extend_from_slice(mapping_id.as_bytes()); hash_bytes(&data) } @@ -309,10 +308,10 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc, input: &Arc) -> (Vec, Vec) { let config = &*app_config.config.load(); match input.input_type { - InputType::M3u => m3u::get_m3u_playlist(client, config, input).await, - InputType::Xtream => xtream::get_xtream_playlist(config, client, input).await, + InputType::M3u => m3u::download_m3u_playlist(client, config, input).await, + InputType::Xtream => xtream::download_xtream_playlist(config, client, input).await, InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![]), - InputType::Library => library::get_library_playlist(client, app_config, input).await, + InputType::Library => library::download_library_playlist(client, app_config, input).await, } } @@ -332,13 +331,25 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so if is_input_enabled(input, &user_targets) { source_downloaded = true; let start_time = Instant::now(); - let (mut playlistgroups, mut error_list) = playlist_download_from_input(client, &app_config, input).await; + // Download playlist for input + let (playlistgroups, mut error_list) = { + let (downloaded_playlist, mut download_err) = playlist_download_from_input(client, &app_config, input).await; + let (playlist, error) = persist_input_playlist(&app_config, input, downloaded_playlist).await; + error!("Failed to persist input playlist {}", input.name); + if let Some(err) = error { + download_err.push(err); + } + (playlist, download_err) + }; + + // Download epg for input let (tvguide, mut tvguide_errors) = if error_list.is_empty() { let working_dir = &app_config.config.load().working_dir; epg::get_xmltv(client, input, working_dir).await } else { (None, vec![]) }; + errors.append(&mut error_list); errors.append(&mut tvguide_errors); let group_count = playlistgroups.len(); @@ -348,7 +359,6 @@ async fn process_source(client: &reqwest::Client, app_config: Arc, so info!("Source is empty {input_name}"); errors.push(notify_err!(format!("Source is empty {input_name}"))); } else { - playlistgroups.iter_mut().for_each(PlaylistGroup::on_load); source_playlists.push( FetchedPlaylist { input, @@ -533,7 +543,7 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { } #[allow(clippy::too_many_arguments)] -async fn process_playlist_for_target(app_config: &AppConfig, +async fn process_playlist_for_target(app_config: &Arc, client: &reqwest::Client, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, diff --git a/backend/src/processing/processor/trakt.rs b/backend/src/processing/processor/trakt.rs index 4904f4432..b68873355 100644 --- a/backend/src/processing/processor/trakt.rs +++ b/backend/src/processing/processor/trakt.rs @@ -36,15 +36,6 @@ fn is_compatible_content_type(cluster: XtreamCluster, content_type: TraktContent } } -/// Extract TMDB ID from playlist item -fn extract_tmdb_id_from_playlist_item(item: &PlaylistItem) -> Option { - if let Some(props) = &item.header.additional_properties { - props.get_tmdb_id() - } else { - None - } -} - fn calculate_year_bonus(playlist_year: Option, trakt_year: Option) -> f64 { if let (Some(p_year), Some(t_year)) = (playlist_year, trakt_year) { if p_year == t_year { @@ -218,7 +209,7 @@ fn match_trakt_items_with_playlist<'a>( if is_compatible_content_type(channel.header.xtream_cluster, list_config.content_type) { let normalized_title = normalize_title_for_matching(&channel.header.title); let channel_year = extract_year_from_title(&channel.header.title); - let channel_tmdb_id = extract_tmdb_id_from_playlist_item(channel); + let channel_tmdb_id = channel.get_tmdb_id(); if let Some(matched) = find_best_match_for_item((channel, normalized_title, channel_year, channel_tmdb_id), &trakt_match_items, list_config) { matches.push(matched); } diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index b6f37e8b0..331a93e4a 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -1,16 +1,7 @@ -use shared::error::{info_err, notify_err}; -use shared::error::{str_to_io_error, TuliproxError}; -use crate::model::{AppConfig, Config, ConfigInput, InputSource}; -use crate::model::{FetchedPlaylist}; -use shared::model::{PlaylistEntry, PlaylistItem, PlaylistItemType, XtreamCluster}; -use crate::repository::storage::get_input_storage_path; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::PathBuf; -use crate::repository::bplustree::BPlusTree; -use crate::repository::storage_const; -use crate::repository::xtream_repository::xtream_get_record_file_path; -use crate::utils; +use shared::error::{info_err}; +use shared::error::{TuliproxError}; +use crate::model::{ConfigInput, InputSource}; +use shared::model::{PlaylistEntry, PlaylistItem, XtreamCluster}; use crate::utils::xtream; pub(in crate::processing) async fn playlist_resolve_download_playlist_item(client: &reqwest::Client, pli: &PlaylistItem, input: &ConfigInput, errors: &mut Vec, resolve_delay: u16, cluster: XtreamCluster) -> Option { @@ -31,74 +22,3 @@ pub(in crate::processing) async fn playlist_resolve_download_playlist_item(clien } result } - -pub(in crate::processing) async fn create_resolve_episode_wal_files(cfg: &Config, input: &ConfigInput) -> Option<(tokio::fs::File, PathBuf)> { - match get_input_storage_path(&input.name, &cfg.working_dir) { - Ok(storage_path) => { - let info_path = storage_path.join(format!("{}.{}", crate::model::XC_FILE_SERIES_EPISODE_RECORD, storage_const::FILE_SUFFIX_WAL)); - let info_file = utils::async_append_or_create_file(&info_path).await.ok()?; - Some((info_file, info_path)) - } - Err(_) => None - } -} - -pub(in crate::processing) async fn create_resolve_info_wal_files(cfg: &Config, input: &ConfigInput, cluster: XtreamCluster) -> Option<(tokio::fs::File, tokio::fs::File, PathBuf, PathBuf)> { - match get_input_storage_path(&input.name, &cfg.working_dir) { - Ok(storage_path) => { - if let Some(file_prefix) = match cluster { - XtreamCluster::Live => None, - XtreamCluster::Video => Some(crate::model::XC_FILE_VOD_INFO), - XtreamCluster::Series => Some(crate::model::XC_FILE_SERIES_INFO) - } { - let content_path = storage_path.join(format!("{file_prefix}_content.{}", storage_const::FILE_SUFFIX_WAL)); - let info_path = storage_path.join(format!("{file_prefix}_record.{}", storage_const::FILE_SUFFIX_WAL)); - let content_file = utils::async_append_or_create_file(&content_path).await.ok()?; - let info_file = utils::async_append_or_create_file(&info_path).await.ok()?; - return Some((content_file, info_file, content_path, info_path)); - } - None - } - Err(_) => None - } -} - -pub(in crate::processing) fn should_update_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap) -> (bool, u32, u64) { - let Some(provider_id) = pli.header.get_provider_id() else { return (false, 0, 0) }; - let last_modified = pli.header.additional_properties.as_ref().and_then(shared::model::StreamProperties::get_last_modified); - let old_timestamp = processed_provider_ids.get(&provider_id); - (old_timestamp.is_none() - || last_modified.is_none() - || *old_timestamp.unwrap() != last_modified.unwrap(), provider_id, last_modified.unwrap_or(0)) -} - -pub(in crate::processing) async fn read_processed_info_ids(cfg: &AppConfig, errors: &mut Vec, fpl: &FetchedPlaylist<'_>, - item_type: PlaylistItemType, extract_ts: F) -> HashMap -where - F: Fn(&V) -> u64, - V: Serialize + for<'de> Deserialize<'de> + Clone, -{ - let mut processed_info_ids = HashMap::new(); - - let fpl_name = &fpl.input.name; - let file_path = match get_input_storage_path(fpl_name, &cfg.config.load().working_dir) - .map(|storage_path| xtream_get_record_file_path(&storage_path, item_type)).and_then(|opt| opt.ok_or_else(|| str_to_io_error("Not supported"))) - { - Ok(file_path) => file_path, - Err(err) => { - errors.push(notify_err!(format!("Could not create storage path for input {fpl_name}: {err}"))); - return processed_info_ids; - } - }; - - { - let file_lock = cfg.file_locks.read_lock(&file_path).await; - if let Ok(info_records) = BPlusTree::::load(&file_path) { - info_records.iter().for_each(|(provider_id, record)| { - processed_info_ids.insert(*provider_id, extract_ts(record)); - }); - } - drop(file_lock); - } - processed_info_ids -} diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 74a059afb..edf04768a 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -2,105 +2,67 @@ use crate::model::{AppConfig, ConfigTarget}; use crate::model::FetchedPlaylist; use crate::processing::parser::xtream::parse_xtream_series_info; use crate::processing::processor::playlist::ProcessingPipe; -use crate::processing::processor::xtream::{create_resolve_episode_wal_files, create_resolve_info_wal_files, playlist_resolve_download_playlist_item, read_processed_info_ids, should_update_info}; -use crate::processing::processor::{create_resolve_options_function_for_xtream_target, handle_error, handle_error_and_return}; -use crate::repository::storage::get_input_storage_path; -use crate::repository::xtream_repository::{write_series_info_to_wal_file, xtream_get_info_file_paths, - xtream_update_input_info_file, xtream_update_input_series_episodes_record_from_wal_file, - xtream_update_input_series_record_from_wal_file}; -use crate::repository::IndexedDocumentReader; -use crate::utils; -use crate::utils::{bincode_serialize, IO_BUFFER_SIZE}; -use log::{error, info, log_enabled, warn, Level}; -use shared::error::{info_err, notify_err}; +use crate::processing::processor::xtream::{playlist_resolve_download_playlist_item}; +use crate::processing::processor::{create_resolve_options_function_for_xtream_target}; +use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; -use shared::model::{InputType, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo}; +use shared::model::{InputType, PlaylistEntry, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo}; use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster}; -use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use std::time::Instant; -use tokio::io::AsyncWriteExt; +use crate::repository::storage::get_input_storage_path; +use crate::repository::xtream_repository::persists_input_series_info; create_resolve_options_function_for_xtream_target!(series); -async fn read_processed_series_info_ids(cfg: &AppConfig, errors: &mut Vec, fpl: &FetchedPlaylist<'_>) -> HashMap { - read_processed_info_ids(cfg, errors, fpl, PlaylistItemType::SeriesInfo, |ts: &u64| *ts).await -} +async fn playlist_resolve_series_info(app_config: &Arc, client: &reqwest::Client, errors: &mut Vec, + fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> Vec { -async fn write_series_episode_record_to_wal_file( - writer: &mut tokio::io::BufWriter<&mut tokio::fs::File>, - provider_id: u32, - additional_properties: Option<&StreamProperties>, -) -> std::io::Result { - if let Some(StreamProperties::Episode(series_episode)) = additional_properties { - if let Ok(content_bytes) = bincode_serialize(&series_episode) { - writer.write_all(&provider_id.to_le_bytes()).await?; - let content_len = content_bytes.len(); - if let Ok(len) = u32::try_from(content_len) { - writer.write_all(&len.to_le_bytes()).await?; - writer.write_all(&content_bytes).await?; - return Ok(content_len + 4usize); - } - error!("Cant write to WAL file, content length exceeds u32"); + let input = fpl.input; + let working_dir = &app_config.config.load().working_dir; + let storage_path = match get_input_storage_path(&input.name, working_dir) { + Ok(storage_path) => storage_path, + Err(err) => { + error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name); + return vec![]; } - } - Ok(0) -} - -async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, errors: &mut Vec, - fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { - // TODO read existing WAL File and import it to avoid duplicate requests - - let mut processed_info_ids: HashMap = read_processed_series_info_ids(cfg, errors, fpl).await; - let mut fetched_in_run: HashSet = HashSet::new(); - // we can't write to the indexed-document directly because of the write lock and time-consuming operation. - // All readers would be waiting for the lock and the app would be unresponsive. - // We collect the content into a wal file and write it once we collected everything. - let Some((mut wal_content_file, mut wal_record_file, wal_content_path, wal_record_path)) - = create_resolve_info_wal_files(&cfg.config.load(), fpl.input, XtreamCluster::Series).await - else { return !processed_info_ids.is_empty(); }; - - let mut content_writer = utils::async_file_writer(&mut wal_content_file); - let mut record_writer = utils::async_file_writer(&mut wal_record_file); - let mut content_updated = false; + }; let series_info_count = fpl.playlistgroups.iter() - .filter(|&plg| plg.xtream_cluster == XtreamCluster::Series) .flat_map(|plg| &plg.channels) - .filter(|&pli| pli.header.item_type == PlaylistItemType::SeriesInfo).count(); + .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Series + && pli.header.item_type == PlaylistItemType::SeriesInfo + && !pli.has_details()).count(); info!("Found {series_info_count} series info to resolve"); let mut last_log_time = Instant::now(); let mut processed_series_info_count = 0; - let mut write_counter = 0usize; - - // LocalSeriesInfo entries are not resolved! + let input = fpl.input; + let mut result: Vec = vec![]; for plg in &mut fpl.playlistgroups { + let mut group_series = vec![]; for pli in &mut plg.channels { - if pli.header.item_type != PlaylistItemType::SeriesInfo { + processed_series_info_count += 1; + if pli.header.xtream_cluster != XtreamCluster::Series + || pli.header.item_type != PlaylistItemType::SeriesInfo + || pli.has_details() { continue; } - let (should_update, provider_id, ts) = should_update_info(pli, &processed_info_ids); - if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) { + let Some(provider_id) = pli.get_provider_id() else { continue; }; + let (group, series_name) = { + let header = &pli.header; + (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) + }; + if provider_id != 0 { if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await { if let Ok(info) = serde_json::from_str::(&content) { let series_stream_props = SeriesStreamProperties::from_info(&info, pli); - handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, &serde_json::to_string(&series_stream_props).unwrap_or_else(|_|String::new()), - &mut content_writer, &mut record_writer).await, - |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}")))); - processed_info_ids.insert(provider_id, ts); - content_updated = true; - write_counter += content.len(); - - // periodic flush to bound BufWriter memory - if write_counter >= IO_BUFFER_SIZE { - write_counter = 0; - if let Err(err) = content_writer.flush().await { - errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); - } - if let Err(err) = record_writer.flush().await { - errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}"))); - } + // the input db needs to be updated + let _ = persists_input_series_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &series_stream_props).await; + // extract episodes from info + if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), &series_stream_props, &group, &series_name, input) { + group_series.extend(episodes.into_iter()); } // Update in-memory playlist items with the newly fetched vod info. @@ -109,116 +71,11 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, } } } - if log_enabled!(Level::Info) { - processed_series_info_count += 1; - if last_log_time.elapsed().as_secs() >= 30 { + if log_enabled!(Level::Info) + && last_log_time.elapsed().as_secs() >= 30 { info!("resolved {processed_series_info_count}/{series_info_count} series info"); last_log_time = Instant::now(); } - } - } - } - info!("resolved {processed_series_info_count}/{series_info_count} series info"); - // content_wal contains the provider_id and series_info with episode listing - // record_wal contains provider_id and timestamp - if content_updated { - handle_error!(content_writer.flush().await, - |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}")))); - handle_error!(record_writer.flush().await, - |err| errors.push(notify_err!(format!("Failed to resolve series tmdb, could not write to wal file {err}")))); - handle_error!(content_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}")))); - handle_error!(record_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info record to wal file {err}")))); - drop(content_writer); - drop(record_writer); - drop(wal_content_file); - drop(wal_record_file); - handle_error!(xtream_update_input_info_file(cfg, fpl.input, &wal_content_path, XtreamCluster::Series).await, - |err| errors.push(err)); - handle_error!(xtream_update_input_series_record_from_wal_file(cfg, fpl.input, &wal_record_path).await, - |err| errors.push(err)); - } - - // we updated now - // - series_info.db which contains the original series_info json - // - series_record.db which contains the series_info provider_id and timestamp - !processed_info_ids.is_empty() -} - -async fn process_series_info( - app_config: &AppConfig, - fpl: &mut FetchedPlaylist<'_>, - errors: &mut Vec, -) -> Vec { - let mut result: Vec = vec![]; - let input = fpl.input; - let config = app_config.config.load(); - let Ok(Some((info_path, idx_path))) = get_input_storage_path(&input.name, &config.working_dir) - .map(|storage_path| xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)) - else { - errors.push(notify_err!("Failed to open input info file for series".to_string())); - return result; - }; - - let mut write_counter = 0usize; - - let _file_lock = app_config.file_locks.read_lock(&info_path).await; - - // Contains the Series Info with episode listing - let Ok(mut info_reader) = IndexedDocumentReader::::new(&info_path, &idx_path) else { return result; }; - - let Some((mut wal_file, wal_path)) = create_resolve_episode_wal_files(&config, input).await else { - errors.push(notify_err!("Could not create wal file for series episodes record".to_string())); - return result; - }; - let mut wal_writer = utils::async_file_writer(&mut wal_file); - - for plg in fpl - .playlistgroups - .iter_mut() - .filter(|plg| plg.xtream_cluster == XtreamCluster::Series) - { - let mut group_series = vec![]; - - // Resolve does not handle LocalSeriesInfo - for pli in plg - .channels - .iter_mut() - .filter(|pli| pli.header.item_type == PlaylistItemType::SeriesInfo) - { - let Some(provider_id) = pli.header.get_provider_id() else { continue; }; - let Ok(content) = info_reader.get(&provider_id) else { continue; }; - if content.is_empty() { - warn!("Series info content is empty, skipping series with provider id: {provider_id}"); - continue; - } - match serde_json::from_str::(&content) { - Ok(series_content) => { - let (group, series_name) = { - let header = &pli.header; - (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) - }; - if let Some(mut series) = parse_xtream_series_info(&series_content, &group, &series_name, input) { - for pli_episode in &mut series { - let Some(provider_id) = &pli_episode.header.get_provider_id() else { continue; }; - match write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, pli.header.additional_properties.as_ref()).await { - Ok(written_bytes) => { - write_counter += written_bytes; - // periodic flush to bound BufWriter memory - if write_counter >= IO_BUFFER_SIZE { - write_counter = 0; - if let Err(err) = wal_writer.flush().await { - errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); - } - } - } - Err(err) => { errors.push(info_err!(format!("Failed to write to series episode wal file: {err}"))) } - } - } - group_series.extend(series.into_iter()); - } - } - Err(err) => errors.push(info_err!(format!("Failed to parse JSON: {err}"))), - } } if !group_series.is_empty() { result.push(PlaylistGroup { @@ -229,19 +86,107 @@ async fn process_series_info( }); } } - - handle_error!(wal_writer.flush().await, |err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}")))); - handle_error!(wal_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}")))); - - drop(wal_writer); - drop(wal_file); - handle_error!(xtream_update_input_series_episodes_record_from_wal_file(app_config, input, &wal_path).await, - |err| errors.push(err)); + info!("resolved {processed_series_info_count}/{series_info_count} series info"); result } +// +// async fn process_series_info( +// app_config: &AppConfig, +// fpl: &mut FetchedPlaylist<'_>, +// errors: &mut Vec, +// ) -> Vec { +// let mut result: Vec = vec![]; +// let input = fpl.input; +// let config = app_config.config.load(); +// let Ok(Some((info_path, idx_path))) = get_input_storage_path(&input.name, &config.working_dir) +// .map(|storage_path| xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)) +// else { +// errors.push(notify_err!("Failed to open input info file for series".to_string())); +// return result; +// }; +// +// let mut write_counter = 0usize; +// +// let _file_lock = app_config.file_locks.read_lock(&info_path).await; +// +// // Contains the Series Info with episode listing +// let Ok(mut info_reader) = IndexedDocumentReader::::new(&info_path, &idx_path) else { return result; }; +// +// let Some((mut wal_file, wal_path)) = create_resolve_episode_wal_files(&config, input).await else { +// errors.push(notify_err!("Could not create wal file for series episodes record".to_string())); +// return result; +// }; +// let mut wal_writer = utils::async_file_writer(&mut wal_file); +// +// for plg in fpl +// .playlistgroups +// .iter_mut() +// .filter(|plg| plg.xtream_cluster == XtreamCluster::Series) +// { +// let mut group_series = vec![]; +// +// // Resolve does not handle LocalSeriesInfo +// for pli in plg +// .channels +// .iter_mut() +// .filter(|pli| pli.header.item_type == PlaylistItemType::SeriesInfo) +// { +// let Some(provider_id) = pli.header.get_provider_id() else { continue; }; +// let Ok(content) = info_reader.get(&provider_id) else { continue; }; +// if content.is_empty() { +// warn!("Series info content is empty, skipping series with provider id: {provider_id}"); +// continue; +// } +// match serde_json::from_str::(&content) { +// Ok(series_content) => { +// let (group, series_name) = { +// let header = &pli.header; +// (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) +// }; +// if let Some(mut series) = parse_xtream_series_info(&series_content, &group, &series_name, input) { +// for pli_episode in &mut series { +// let Some(provider_id) = &pli_episode.header.get_provider_id() else { continue; }; +// match write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, pli.header.additional_properties.as_ref()).await { +// Ok(written_bytes) => { +// write_counter += written_bytes; +// // periodic flush to bound BufWriter memory +// if write_counter >= IO_BUFFER_SIZE { +// write_counter = 0; +// if let Err(err) = wal_writer.flush().await { +// errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); +// } +// } +// } +// Err(err) => { errors.push(info_err!(format!("Failed to write to series episode wal file: {err}"))) } +// } +// } +// group_series.extend(series.into_iter()); +// } +// } +// Err(err) => errors.push(info_err!(format!("Failed to parse JSON: {err}"))), +// } +// } +// if !group_series.is_empty() { +// result.push(PlaylistGroup { +// id: plg.id, +// title: plg.title.clone(), +// channels: group_series, +// xtream_cluster: XtreamCluster::Series, +// }); +// } +// } +// +// handle_error!(wal_writer.flush().await, |err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}")))); +// handle_error!(wal_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}")))); +// +// drop(wal_writer); +// drop(wal_file); +// handle_error!(xtream_update_input_series_episodes_record_from_wal_file(app_config, input, &wal_path).await, +// |err| errors.push(err)); +// result +// } - -pub async fn playlist_resolve_series(cfg: &AppConfig, +pub async fn playlist_resolve_series(cfg: &Arc, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, @@ -252,8 +197,7 @@ pub async fn playlist_resolve_series(cfg: &AppConfig, let (resolve_series, resolve_delay) = get_resolve_series_options(target, processed_fpl); if !resolve_series { return; } - if !playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_delay).await { return; } - let series_playlist = process_series_info(cfg, provider_fpl, errors).await; + let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_delay).await; if series_playlist.is_empty() { return; } // original content saved into original list for plg in &series_playlist { diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 193e30644..4d4c3fa64 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -1,131 +1,68 @@ +use std::sync::Arc; use crate::model::FetchedPlaylist; use crate::model::{AppConfig, ConfigTarget}; -use crate::processing::processor::xtream::{create_resolve_info_wal_files, playlist_resolve_download_playlist_item, read_processed_info_ids, should_update_info}; -use crate::processing::processor::{create_resolve_options_function_for_xtream_target, handle_error, handle_error_and_return}; -use crate::repository::xtream_repository::{write_vod_info_to_wal_file, xtream_update_input_info_file, xtream_update_input_vod_record_from_wal_file, InputVodInfoRecord}; -use crate::utils; -use crate::utils::IO_BUFFER_SIZE; -use log::{info, log_enabled, Level}; -use shared::error::notify_err; +use crate::processing::processor::xtream::{playlist_resolve_download_playlist_item}; +use crate::processing::processor::{create_resolve_options_function_for_xtream_target}; +use crate::repository::xtream_repository::{persists_input_vod_info}; +use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; use shared::model::{InputType, PlaylistEntry, StreamProperties, VideoStreamProperties, XtreamVideoInfo}; use shared::model::{PlaylistItemType, XtreamCluster}; -use std::collections::{HashMap, HashSet}; use std::time::Instant; -use tokio::io::AsyncWriteExt; +use crate::repository::storage::get_input_storage_path; create_resolve_options_function_for_xtream_target!(vod); -async fn read_processed_vod_info_ids(cfg: &AppConfig, errors: &mut Vec, fpl: &FetchedPlaylist<'_>) -> HashMap { - read_processed_info_ids(cfg, errors, fpl, PlaylistItemType::Video, |record: &InputVodInfoRecord| record.ts).await -} - -pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>) { +pub async fn playlist_resolve_vod(app_config: &Arc, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>) { let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl); if !resolve_movies { return; } - // TODO read existing WAL File and import it to avoid duplicate requests - - // we cant write to the indexed-document directly because of the write lock and time-consuming operation. - // All readers would be waiting for the lock and the app would be unresponsive. - // We collect the content into a wal file and write it once we collected everything. - let config = app_config.config.load(); - let Some((mut wal_content_file, mut wal_record_file, wal_content_path, wal_record_path)) - = create_resolve_info_wal_files(&config, fpl.input, XtreamCluster::Video).await - else { return; }; - - let mut processed_info_ids: HashMap = read_processed_vod_info_ids(app_config, errors, fpl).await; - let mut fetched_in_run: HashSet = HashSet::new(); - let mut content_writer = utils::async_file_writer(&mut wal_content_file); - let mut record_writer = utils::async_file_writer(&mut wal_record_file); - let mut content_updated = false; + let input = fpl.input; + let working_dir = &app_config.config.load().working_dir; + let storage_path = match get_input_storage_path(&input.name, working_dir) { + Ok(storage_path) => storage_path, + Err(err) => { + error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name); + return; + } + }; // LocalVideo entries are not resolved! - - // TODO merge both filters to one let vod_info_count = fpl.playlistgroups.iter() .flat_map(|plg| &plg.channels) - .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Video && pli.header.item_type == PlaylistItemType::Video).count(); + .filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video + && pli.header.item_type == PlaylistItemType::Video + && !pli.has_details()).count(); + info!("Found {vod_info_count} vod info to resolve"); let mut last_log_time = Instant::now(); let mut processed_vod_info_count = 0; - let mut write_counter = 0usize; for plg in &mut fpl.playlistgroups { for pli in &mut plg.channels { - // LocalVideo files are not resolved - if pli.header.item_type != PlaylistItemType::Video { + processed_vod_info_count += 1; + if pli.header.xtream_cluster != XtreamCluster::Video + || pli.header.item_type != PlaylistItemType::Video + || pli.has_details() { continue; } - let (should_update, provider_id, _ts) = should_update_info(pli, &processed_info_ids); - if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) { - if pli.get_item_type().is_local() { - continue; - } + let Some(provider_id) = pli.get_provider_id() else { continue; }; + if provider_id != 0 { if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await { if let Ok(info) = serde_json::from_str::(&content) { - let provider_id = info.movie_data.stream_id; - let added = info.movie_data.added.parse::().unwrap_or(0); - let tmdb_id = info.info.tmdb_id.parse::().unwrap_or(0); - let info_record = InputVodInfoRecord { - tmdb_id, - ts: added, - release_date: info.info.releasedate.clone(), - }; - let video_stream_props = VideoStreamProperties::from_info(&info, pli); - handle_error_and_return!(write_vod_info_to_wal_file(provider_id, &serde_json::to_string(&video_stream_props).unwrap_or_else(|_|String::new()), - &info_record, &mut content_writer, &mut record_writer).await, - |err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}")))); - processed_info_ids.insert(provider_id, added); - content_updated = true; - write_counter += content.len(); - // periodic flush to bound BufWriter memory - if write_counter >= IO_BUFFER_SIZE { - write_counter = 0; - if let Err(err) = content_writer.flush().await { - errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); - } - if let Err(err) = record_writer.flush().await { - errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}"))); - } - } - - // Update in-memory playlist items with the newly fetched vod info. + let _ = persists_input_vod_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &video_stream_props).await; // This makes the data available for subsequent processing steps like STRM export. pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props))); } } } - if log_enabled!(Level::Info) { - processed_vod_info_count += 1; - if last_log_time.elapsed().as_secs() >= 30 { - info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); - last_log_time = Instant::now(); - } + if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); + last_log_time = Instant::now(); } } } info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); - if content_updated { - // TODO better approach for transactional updates is multiplexed WAL file. - // final flush & sync with proper error handling - handle_error!(content_writer.flush().await, - |err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}")))); - handle_error!(record_writer.flush().await, - |err| errors.push(notify_err!(format!("Failed to resolve vod tmdb, could not write to wal file {err}")))); - handle_error!(content_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync vod info to wal file {err}")))); - handle_error!(record_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync vod info record to wal file {err}")))); - // drop writers and files to release handles - drop(content_writer); - drop(record_writer); - drop(wal_content_file); - drop(wal_record_file); - - handle_error!(xtream_update_input_info_file(app_config, fpl.input, &wal_content_path, XtreamCluster::Video).await, - |err| errors.push(err)); - handle_error!(xtream_update_input_vod_record_from_wal_file(app_config, fpl.input, &wal_record_path).await, - |err| errors.push(err)); - } } diff --git a/backend/src/ptt/constants.rs b/backend/src/ptt/constants.rs new file mode 100644 index 000000000..418138ae3 --- /dev/null +++ b/backend/src/ptt/constants.rs @@ -0,0 +1,83 @@ +use std::sync::LazyLock; +use regex::Regex; +use fancy_regex::{Regex as FancyRegex}; + + +pub const BRACKETS: &[(&str, &str)] = &[("{", "}"), ("[", "]"), ("(", ")")]; + +pub const NON_ENGLISH_RANGES: &str = "\u{3040}-\u{30ff}\u{3400}-\u{4dbf}\u{4e00}-\u{9fff}\u{f900}-\u{faff}\u{ff66}-\u{ff9f}\u{0400}-\u{04ff}\u{0600}-\u{06ff}\u{0750}-\u{077f}\u{0c80}-\u{0cff}\u{0d00}-\u{0d7f}\u{0e00}-\u{0e7f}"; + +pub struct PttConstants { + pub movie_regex: Regex, + pub russian_cast_regex_fancy: FancyRegex, + pub alt_titles_regex: Regex, + pub before_title_match_regex: Regex, + pub not_only_non_english_regex: FancyRegex, + pub not_allowed_symbols_at_start_and_end: FancyRegex, + pub remaining_not_allowed_symbols_at_start_and_end: Regex, + pub redundant_symbols_at_end: Regex, + pub empty_brackets_regex: Regex, + pub parantheses_without_content: Regex, + pub star_regex_1: Regex, + pub star_regex_2: Regex, + pub mp3_regex: Regex, + pub spacing_regex: Regex, + pub special_char_spacing: Regex, + pub sub_pattern: Regex, + pub integer: Regex, + pub word: Regex, + pub dot: Regex, + pub months: Vec<(Regex, &'static str)>, +} + +pub static PTT_CONSTANTS: LazyLock = LazyLock::new(|| { + PttConstants { + movie_regex: Regex::new(r"(?i)[\[(]movie[)\]]").unwrap(), + russian_cast_regex_fancy: FancyRegex::new( + r"(?i)\([^)]*[\u0400-\u04ff][^)]*\)$|(?<=\/.*)\(.*\)$", + ) + .unwrap(), + alt_titles_regex: Regex::new(&format!( + r"(?i)[^/|(]*[{NON_ENGLISH_RANGES}][^/|]*[/|]|[/|][^/|(]*[{NON_ENGLISH_RANGES}][^/|]*" + )) + .unwrap(), + before_title_match_regex: Regex::new(r"^(?:\[([^\]]+)\])").unwrap(), + not_only_non_english_regex: FancyRegex::new(&format!( + r"(?i)([a-zA-Z][^{NON_ENGLISH_RANGES}]+)([{NON_ENGLISH_RANGES}].*[{NON_ENGLISH_RANGES}])|[{NON_ENGLISH_RANGES}].*[{NON_ENGLISH_RANGES}](?=[^{NON_ENGLISH_RANGES}]+[a-zA-Z])" + )).unwrap(), + not_allowed_symbols_at_start_and_end: FancyRegex::new(&format!( + r"(?i)^[^\w{NON_ENGLISH_RANGES}#\x5B【★]+|[ \-:/\x5C\x5B|{{(#$&^]+$" + )) + .unwrap(), + remaining_not_allowed_symbols_at_start_and_end: Regex::new( + &format!(r"^[^\w{NON_ENGLISH_RANGES}#]+|]$") + ) + .unwrap(), + redundant_symbols_at_end: Regex::new(r"[ \-:./\\]+$").unwrap(), + empty_brackets_regex: Regex::new(r"\(\s*\)|\[\s*\]|\{\s*\}").unwrap(), + parantheses_without_content: Regex::new(r"\(\W*\)|\[\W*\]|\{\W*\}").unwrap(), + star_regex_1: Regex::new(r"^[\[【★].*[\]】★][ .]?(.+)").unwrap(), + star_regex_2: Regex::new(r"(.+)[ .]?[\[【★].*[\]】★]$").unwrap(), + mp3_regex: Regex::new(r"(?i)\bmp3$").unwrap(), + spacing_regex: Regex::new(r"\s+").unwrap(), + special_char_spacing: Regex::new(r"[\-\+\_\{\}\[\]]\W{2,}").unwrap(), + sub_pattern: Regex::new(r"_+").unwrap(), + integer: Regex::new(r"\d+").unwrap(), + word: Regex::new(r"\W+").unwrap(), + dot: Regex::new(r"\.").unwrap(), + months: vec! [ + (Regex::new(r"(?i)\bJanu\b").unwrap(), "Jan"), + (Regex::new(r"(?i)\bFebr\b").unwrap(), "Feb"), + (Regex::new(r"(?i)\bMarc\b").unwrap(), "Mar"), + (Regex::new(r"(?i)\bApri\b").unwrap(), "Apr"), + (Regex::new(r"(?i)\bMay\b").unwrap(), "May"), + (Regex::new(r"(?i)\bJune\b").unwrap(), "Jun"), + (Regex::new(r"(?i)\bJuly\b").unwrap(), "Jul"), + (Regex::new(r"(?i)\bAugu\b").unwrap(), "Aug"), + (Regex::new(r"(?i)\bSept\b").unwrap(), "Sep"), + (Regex::new(r"(?i)\bOcto\b").unwrap(), "Oct"), + (Regex::new(r"(?i)\bNove\b").unwrap(), "Nov"), + (Regex::new(r"(?i)\bDece\b").unwrap(), "Dec"), + ] + } +}); diff --git a/backend/src/ptt/handlers.rs b/backend/src/ptt/handlers.rs new file mode 100644 index 000000000..a2df2c5ad --- /dev/null +++ b/backend/src/ptt/handlers.rs @@ -0,0 +1,4708 @@ +use crate::ptt::parser::{HandlerOptions, MatchInfo, ParseContext, PttParser}; +use crate::ptt::transformers::{boolean, date, first_uinteger, lowercase, none, range_i32, range_u32, transform_resolution, uinteger, uppercase, value}; +use fancy_regex::Regex as FancyRegex; + +fn parse_season_range(val: &str) -> Vec { + let nums: Vec = val + .split(|c: char| !c.is_numeric()) + .filter_map(|s| s.parse::().ok()) + .collect(); + + if nums.len() == 2 { + let start = nums[0]; + let end = nums[1]; + if start < end && (end - start) < 100 { + let lower = val.to_lowercase(); + if val.contains('-') + || lower.contains("to") + || lower.contains("thru") + || val.contains(':') + { + return (start..=end).collect(); + } + } + } + nums +} + +#[allow(clippy::too_many_lines)] +pub fn add_defaults(parser: &mut PttParser) { + parser.add_handler( + "tmdb", + FancyRegex::new(r"(?i)\btmdb\b[-=]\d+").unwrap(), + first_uinteger, + |meta, val| { + if let Some(v) = val { + meta.tmdb = Some(v); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + parser.add_handler( + "tvdb", + FancyRegex::new(r"(?i)\btvdb\b[-=]\d+").unwrap(), + first_uinteger, + |meta, val| { + if let Some(v) = val { + meta.tvdb = Some(v); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + + parser.add_handler( + "quality", + FancyRegex::new(r"(?i)\bPRE[- .]?HDRip\b").unwrap(), + boolean, + |meta, val| { + meta.trash = val; + if val { + meta.quality = Some("SCR".to_string()); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bE[- ]?Sub\b").unwrap(), + |_| "en".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "quality", + FancyRegex::new(r"(?i)\bTS-Screener\b").unwrap(), + boolean, + |meta, val| { + meta.trash = val; + if val { + meta.quality = Some("TeleSync".to_string()); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + + parser.add_handler( + "year", + FancyRegex::new(r"\b19\d{2}\s?-\s?20\d{2}\b").unwrap(), + first_uinteger, + |meta, val| { + if let Some(v) = val { + meta.year = Some(v); + } + }, + HandlerOptions { + remove: false, + ..Default::default() + }, + ); + + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)\b(?:19|20)\d{2}\s*[-]\s*(?:(?:19|20)\d{2}|\d{2})\b").unwrap(), + none, + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)\b100[ .-]*years?[ .-]*quest\b").unwrap(), + none, + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)\[?(\+.)?Extras\]?").unwrap(), + none, + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)(\+Movies)?\+Specials").unwrap(), + none, + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "group", + FancyRegex::new(r"-?EDGE2020").unwrap(), + |_| "EDGE2020".to_string(), + |meta, val| meta.group = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)TV Money").unwrap(), + none, + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + parser.add_handler( + "container", + FancyRegex::new(r"(?i)\.?[\[(]?\b(MKV|AVI|MP4|WMV|MPG|MPEG)\b[\])]?").unwrap(), + lowercase, + |meta, val| meta.container = Some(val), + HandlerOptions::default(), + ); + + parser.add_handler( + "torrent", + FancyRegex::new(r"\.torrent$").unwrap(), + boolean, + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + parser.add_handler( + "adult", + FancyRegex::new(r"\b(XXX|xxx|Xxx)\b").unwrap(), + boolean, + |meta, val| meta.adult = val, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + if let Ok(re) = FancyRegex::new(r"(?i)\b(18\+|adult|porn|xxx)\b") { + parser.add_handler( + "adult", + re, + boolean, + |meta, _| meta.adult = true, + HandlerOptions::default(), + ); + } + + parser.add_handler( + "extras", + FancyRegex::new(r"(?i)\bOVA\b").unwrap(), + |_| "OVA".to_string(), + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + parser.add_handler( + "extras", + FancyRegex::new(r"(?i)\bOVA\b").unwrap(), + |_| "OVA".to_string(), + |_, _| {}, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?3840x\d{4}[\])?]?").unwrap(), + |_| "2160p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?1920x\d{3,4}[\])?]?").unwrap(), + |_| "1080p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?1280x\d{3}[\])?]?").unwrap(), + |_| "720p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?(\d{3,4}x\d{3,4})[\])?]?p?").unwrap(), + |val: &str| format!("{val}p"), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(480|720|1080)0[pi]").unwrap(), + |val: &str| format!("{val}p"), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:QHD|QuadHD|WQHD|2560(\d+)?x(\d+)?1440p?)").unwrap(), + |_| "1440p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:Full HD|FHD|1920(\d+)?x(\d+)?1080p?)").unwrap(), + |_| "1080p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)(2160p?|4k)").unwrap(), + |_| "2160p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)1080p?").unwrap(), + |_| "1080p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)720p?").unwrap(), + |_| "720p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)480p?").unwrap(), + |_| "480p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\b(?:4k|2160p|1080p|720p|480p)(?!.*\b(?:4k|2160p|1080p|720p|480p)\b)") + .unwrap(), + transform_resolution, + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\b4k|21600?[pi]\b").unwrap(), + |_| "2160p".to_string(), + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(\d{3,4}[pi])").unwrap(), + lowercase, + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(240|360|480|576|720|1080|2160|3840)[pi]").unwrap(), + lowercase, + |meta, val| meta.resolution = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + + parser.add_handler( + "episode_code", + FancyRegex::new(r"[\[\()]([A-Fa-f0-9]{8})[\]\)]").unwrap(), + uppercase, + |meta, val| meta.episode_code = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "episode_code", + FancyRegex::new(r"[\[\()]([0-9]{8})[\]\)]").unwrap(), + uppercase, + |meta, val| meta.episode_code = Some(val), + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + + parser.add_handler( + "trash", + FancyRegex::new(r"(?i)\b(?:H[DQ][ .-]*)?(?() + .unwrap() + }, + |meta, val| meta.year = Some(val), + HandlerOptions { + remove: false, + ..Default::default() + }, + ); + parser.add_handler( + "channels", + FancyRegex::new(r"(?i)\b(?:x[2-4]|5[\W]1(?:x[2-4])?)\b").unwrap(), + |_| "5.1".to_string(), + |meta, val| { + if !meta.channels.contains(&val) { + meta.channels.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "channels", + FancyRegex::new(r"(?i)\b7[\.\- ]1(.?ch(annel)?)?\b").unwrap(), + |_| "7.1".to_string(), + |meta, val| { + if !meta.channels.contains(&val) { + meta.channels.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "channels", + FancyRegex::new(r"(?i)\+?2[\.\s]0(?:x[2-4])?\b").unwrap(), + |_| "2.0".to_string(), + |meta, val| { + if !meta.channels.contains(&val) { + meta.channels.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(?!.+HR)(DTS.?HD.?Ma(ster)?|DTS.?X)\b").unwrap(), + |_| "DTS Lossless".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bDTS(?!(.?HD.?Ma(ster)?|.X)).?(HD.?HR|HD)?\b").unwrap(), + |_| "DTS Lossy".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(Dolby.?)?Atmos\b").unwrap(), + |_| "Atmos".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(True[ .-]?HD|\.True\.)\b").unwrap(), + |_| "TrueHD".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + skip_from_title: true, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bTRUE\b").unwrap(), + |_| "TrueHD".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + skip_from_title: true, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bFLAC(?:\d+(?:\.\d+)?)?(?:x\d+)?").unwrap(), + |_| "FLAC".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)DD2?[\+p]|DD Plus|Dolby Digital Plus|DDP(5[ \.\_]1)?|E-?AC-?3(?:-S\d+)?") + .unwrap(), + |_| "Dolby Digital Plus".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bddp(5.1)?").unwrap(), + |_| "Dolby Digital Plus".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bMP3\b").unwrap(), + |_| "MP3".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(DD|Dolby.?Digital|DolbyD|AC-?3(x2)?(?:-S\d+)?)\b").unwrap(), + |_| "Dolby Digital".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bQ?Q?AAC(x?2)?\b").unwrap(), + |_| "AAC".to_string(), + |meta, val| { + if !meta.audio.contains(&val) { + meta.audio.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + + parser.add_handler( + "group", + FancyRegex::new(r"(?i)- ?(?!\d+$|S\d+|\d+x|ep?\d+|[^\[]+]$)([^\-. \[]+[^\-. \[)\]\d][^\-. \[)\]]*)(?:\[[\w.-]+])?(?=\.\w{2,4}$|$)").unwrap(), + value, + |meta, val| meta.group = Some(val), + HandlerOptions { remove: false, ..Default::default() }, + ); + parser.add_handler( + "group", + FancyRegex::new(r"\(([\w-]+)\)(?:$|\.\w{2,4}$)").unwrap(), + value, + |meta, val| meta.group = Some(val), + HandlerOptions::default(), + ); + parser.add_handler( + "group", + FancyRegex::new(r"^\[([^\[\]]+)\]").unwrap(), + value, + |meta, val| meta.group = Some(val), + HandlerOptions::default(), + ); + + parser.add_handler( + "volumes", + FancyRegex::new(r"(?i)\bvol(?:s|umes?)?[. -]*(?:\d{1,2}[., +/\\&-]+)+\d{1,2}\b").unwrap(), + range_i32, + |meta, val| { + if let Some(v) = val { + meta.volumes = v; + } + }, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + let volume_regex = FancyRegex::new(r"(?i)\bvol(?:ume)?[. -]*(\d{1,2})\b").unwrap(); + parser.add_handler_fn( + "volumes", + Box::new(move |context: &mut ParseContext| -> Option { + let title = &context.title; + let matched = &context.matched; + + let start_index = matched.get("year").map_or(0, |m| m.match_index); + + if start_index >= title.len() { + return None; + } + + let search_slice = &title[start_index..]; + + if let Ok(Some(m)) = volume_regex.find(search_slice) { + let raw_match = m.as_str().to_string(); + let relative_start = m.start(); + + if let Ok(Some(cap)) = volume_regex.captures(search_slice) { + let volume_number = cap + .get(1) + .map_or(0, |m| m.as_str().parse::().unwrap_or(0)); + + context.result.volumes = vec![volume_number]; + } + + let abs_start = start_index + relative_start; + + let info = MatchInfo { + raw_match, + match_index: abs_start, + remove: true, + skip_from_title: false, + }; + + context.matched.insert("volumes".to_string(), info.clone()); + return Some(info); + } + None + }), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)((?:s\d{1,2}[., +/\\&-]+)+s\d{1,2}\b)") + .unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)[(\[]?(s\d{2,}-\d{2,}\b)[)\]]?").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)[(\[]?(s[1-9]-[2-9])[)\]]?").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)\d+ª(?:.+)?(?:a.?)?\d+ª(?:(?:.+)?(?:temporadas?))").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:seasons?|[Сс]езони?|temporadas?)[. ]?[-:]?[. ]?[( \[]?((?:\d{1,2}[., /\\&]+)+\d{1,2}\b)[)\]]?").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { remove: true, ..Default::default() }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:seasons?|[Сс]езони?|temporadas?)[. ]?[-:]?[. ]?[( \[]?((?:\d{1,2}[.-]+)+[1-9]\d?\b)(?!\W*\d{4})[)\]]?").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { remove: true, ..Default::default() }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?season[. ]?[( \[]?((?:\d{1,2}[. -]+)+[1-9]\d?\b)[)\]]?(?!.*\.\w{2,4}$)").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { remove: true, ..Default::default() }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?\bseasons?\b[. -]?(\d{1,2}[. -]?(?:to|thru|and|\+|:)[. -]?\d{1,2})\b").unwrap(), + parse_season_range, + |meta, val| meta.seasons.extend(val), + HandlerOptions { remove: true, ..Default::default() }, + ); + parser.add_handler( + "quality", + FancyRegex::new(r"(?i)\bDVB(?:\b|-)").unwrap(), + |_| "HDTV".to_string(), + |meta, val| meta.quality = Some(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:saison|seizoen|season|series|temp(?:orada)?):?[. ]?(\d{1,2})\b").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + |meta, val| meta.seasons.extend(val), + HandlerOptions { remove: true, ..Default::default() }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(\d{1,2})(?:-?й)?[. _]?(?:[Сс]езон|sez(?:on)?)(?:\W?\D|$)").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)[Сс]езон:?[. _]?№?(\d{1,2})(?!\d)").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:\D|^)(\d{1,2})Â?[°ºªa]?[. ]*temporada").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)t(\d{1,3})(?:[ex]+|$)").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete)?(?().unwrap_or(0)], + |meta, val| meta.seasons.extend(val), + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + + parser.add_handler( + "episodes", + FancyRegex::new( + r"(?i)(?:[\W\d]|^)e[ .]?[\[(]?(\d{1,3}(?:[ .-]*(?:[&+]|e|.){1,2}(?:[ .]*e)?[ .]?\d{1,3})+)(?:\W|$)", + ).unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:[\W\d]|^)ep[ .]?[\[(]?(\d{1,3}(?:[ .-]*(?:[&+]|ep){1,2}[ .]?\d{1,3})+)(?:\W|$)").unwrap(), + range_u32, + |meta, val| if let Some(v) = val { meta.episodes.extend(v) }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new( + r"(?i)(?:[\W\d]|^)\d+[xх][ .]?[\[(]?(\d{1,3}(?:[ .]?[xх][ .]?\d{1,3})+)(?:\W|$)", + ).unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)Серии:\s+(\d+)\s+(?:of|из|iz)\s+\d+\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:[\W\d]|^)(?:episodes?|[Сс]ерии:?)[ .]?[\[(]?(\d{1,3}(?:[ .+]*[&+][ .]?\d{1,3})+)(?:\W|$)").unwrap(), + range_u32, + |meta, val| if let Some(v) = val { meta.episodes.extend(v) }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)[\[(]?(?:\D|^)(\d{1,3}[ .]?ao[ .]?\d{1,3})[)\]]?(?:\W|$)").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:[\W\d]|^)(?:e|eps?|episodes?|[Сс]ерии:?|\d+[xх])[ .]*[\[(]?(\d{1,3}(?:-\d{1,3})+)(?:\W|$)").unwrap(), + range_u32, + |meta, val| if let Some(v) = val { meta.episodes.extend(v) }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\W|^)(\d{1,3}(?:[ .]*~[ .]*\d{1,3})+)(?:\W|$)").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\bE\d{1,4}\s*à\s*E\d{1,4}\b").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)[st]\d{1,2}[. ]?[xх-]?[. ]?(?:e|x|х|ep|-|\.)[. ]?(\d{1,4})(?:[abc]|v0?[1-4]|\D|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| if let Some(v) = val { meta.episodes.extend(v) }, + HandlerOptions { remove: true, ..Default::default() }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\b[st]\d{2}(\d{2})\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)-\s(\d{1,3}[ .]*-[ .]*\d{1,3})(?!-\d)(?:\W|$)").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)s\d{1,2}\s?\((\d{1,3}[ .]*-[ .]*\d{1,3})\)").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?:^|/)\d{1,2}-(\d{2})\b(?!-\d)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?<=^\[.+].+)[. ]+-[. ]+(\d{1,4})[. ]+(?=\W)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)Ep.\d+.-.\\d+").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\b[ée]p?(?:isode)?|[Ээ]пизод|[Сс]ер(?:ии|ия|\.)?|cap(?:itulo)?|epis[oó]dio)[. ]?[-:#№]?[. ]?(\d{1,4})(?:[abc]|v0?[1-4]|\W|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| if let Some(v) = val { meta.episodes.extend(v) }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\b(\d{1,3})(?:-?я)?[ ._-]*(?:ser(?:i?[iyja]|\b)|[Сс]ер(?:ии|ия|\.)?)") + .unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?:\D|^)\d{1,2}[. ]?[xх][. ]?(\d{1,3})(?:[abc]|v0?[1-4]|\D|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?<=S\d{2}E)(\d+)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"[\[(]\d{1,2}\.(\d{1,3})[)\]]").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"\b[Ss]\d{1,2}[ .](\d{1,2})\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"-\s?\d{1,2}\.(\d{2,3})\s?-").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\[|\()(\d+)\s(?:of|из|iz)\s\d+(?:\]|\))").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?<=\D|^)(\d{1,3})[. ]?(?:of|из|iz)[. ]?\d{1,3}(?=\D|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"\b\d{2}[ ._-](\d{2})(?:.F)?\.\\w{2,4}$").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(\d+)(?=.?\[([A-Z0-9]{8})\])").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions::default(), + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\W|^)(?:\d+)?(?:e|ep)(\d{1,3})(?:\W|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)E(\d+)\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: false, + skip_if_already_found: true, + ..Default::default() + }, + ); + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\b(\d{1,4})-(\d{1,4})\b").unwrap(), + range_u32, + |meta, val| { + if let Some(v) = val { + meta.episodes.extend(v); + } + }, + HandlerOptions { + remove: false, + skip_if_already_found: true, + ..Default::default() + }, + ); + + parser.add_handler( + "country", + FancyRegex::new(r"\b(US|UK|AU|NZ|CA)\b").unwrap(), + value, + |meta, val| meta.country = Some(val), + HandlerOptions::default(), + ); + + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bengl?(?:sub[A-Z]*)?\b").unwrap(), + |_| "en".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bEnglish[\. _-]*(?:subs?|sdh|hi)\b").unwrap(), + |_| "en".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:ingl[eéê]s|inglese?)\b").unwrap(), + |_| "en".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\[En\b").unwrap(), + |_| "en".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bIT\s+EN\b").unwrap(), + |_| "it".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + meta.languages.push("en".to_string()); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bEng(?:,|\s)").unwrap(), + |_| "en".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bCze(?:ch)?\b").unwrap(), + |_| "cs".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bGer(?:,|\s|\b)").unwrap(), + |_| "de".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\[spanish\]").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:español|espanhol)\b").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bFR(?:a|e|anc[eê]s|VF[FQIB2]?)\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b\[?(VF[FQRIB2]?\]?\b|(VOST)?FR2?)\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:GERMAN|GER)\b|(?-i)\bDE\b").unwrap(), + |_| "de".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(TRUE|SUB).?FRENCH\b|\bFRENCH\b|\bFre?\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_first: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(VOST(?:FR?|A)?)\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(VF[FQIB2]?|(TRUE|SUB).?FRENCH|(VOST)?FR2?)\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: true, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bspanish\W?latin|american\W*(?:spa|esp?)").unwrap(), + |_| "la".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:\bla\b.+(?:cia\b))").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:audio.)?lat(?:in?|ino)?\b").unwrap(), + |_| "la".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:audio.)?(?:ESP?|spa|(en[ .]+)?espa[nñ]ola?|castellano)\b").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bes(?=[ .,/-]+(?:[A-Z]{2}[ .,/-]+){2,})\b").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?<=[ .,/-]+(?:[A-Z]{2}[ .,/-]+){2,})es\b").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?<=[ .,/-]+[A-Z]{2}[ .,/-]+)es(?=[ .,/-]+[A-Z]{2}[ .,/-]+)\b").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bes(?=\.(?:ass|ssa|srt|sub|idx)$)").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(temporadas?|completa)\b").unwrap(), + |_| "es".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:INT[EÉ]GRALE?)\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: false, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:Saison)\b").unwrap(), + |_| "fr".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: false, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:p[rt]|en|port)[. (\\/-]*BR\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bbr(?:a|azil|azilian)\W+(?:pt|por)\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + remove: true, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:leg(?:endado|endas?)?|dub(?:lado)?|portugu[eèê]se?)[. -]*BR\b") + .unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bleg(?:endado|endas?)\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bportugu[eèê]s[ea]?\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bPT[. -]*(?:PT|ENG?|sub(?:s|titles?))\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bpt(?=\.(?:ass|ssa|srt|sub|idx)$)").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bPT\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bpor\b").unwrap(), + |_| "pt".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + skip_from_title: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b-?ITA\b").unwrap(), + |_| "it".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(? Option { + let title = &context.title; + let matched = &context.matched; + + let start_index = matched.get("year").map_or(0, |m| m.match_index); + + if start_index >= title.len() { + return None; + } + + let search_slice = &title[start_index..]; + + if let Ok(Some(m)) = volume_regex.find(search_slice) { + let raw_match = m.as_str().to_string(); + let relative_start = m.start(); + + if let Ok(Some(cap)) = volume_regex.captures(search_slice) { + let volume_number = cap + .get(1) + .map_or(0, |m| m.as_str().parse::().unwrap_or(0)); + + context.result.volumes = vec![volume_number]; + } + + let abs_start = start_index + relative_start; + + let info = MatchInfo { + raw_match, + match_index: abs_start, + remove: true, + skip_from_title: false, + }; + + context.matched.insert("volumes".to_string(), info.clone()); + return Some(info); + } + None + }), + ); + let ep_regex = FancyRegex::new(r"\b\d{1,4}\b").unwrap(); + + parser.add_handler_fn( + "episodes", + Box::new(move |context: &mut ParseContext| -> Option { + if context.matched.contains_key("episodes") { + return None; + } + + let title = &context.title; + + if anime_regex.is_match(title).unwrap_or(false) { + if let Ok(Some(m)) = ep_regex.find(title) { + let raw_match = m.as_str().to_string(); + let val = raw_match.parse::().unwrap_or(0); + + context.result.episodes.push(val); + + let info = MatchInfo { + raw_match, + match_index: m.start(), + remove: true, + skip_from_title: true, + }; + context.matched.insert("episodes".to_string(), info.clone()); + return Some(info); + } + } + None + }), + ); + + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(PLDUB|PLSUB|DUBPL|DubbingPL|LekPL|LektorPL)\b").unwrap(), + |_| "pl".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(PLDUB|PLSUB|DUBPL|DubbingPL|LekPL|LektorPL)\b").unwrap(), + |_| "pl".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(PLDUB|PLSUB|DUBPL|DubbingPL|LekPL|LektorPL)\b").unwrap(), + |_| "pl".to_string(), + |meta, val| { + if !meta.languages.contains(&val) { + meta.languages.push(val); + } + }, + HandlerOptions { + remove: true, + skip_if_already_found: false, + ..Default::default() + }, + ); + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\b(?:(? = LazyLock::new(|| { + let mut parser = PttParser::new(); + handlers::add_defaults(&mut parser); + parser +}); + +pub fn ptt_parse_title(title: &str) -> PttMetadata { + PTT_PARSER.parse(title, false) +} diff --git a/backend/src/ptt/models.rs b/backend/src/ptt/models.rs new file mode 100644 index 000000000..b3035ae3f --- /dev/null +++ b/backend/src/ptt/models.rs @@ -0,0 +1,95 @@ +use serde::{Deserialize, Serialize}; + +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] +pub struct PttMetadata { + pub title: String, + pub seasons: Vec, + pub episodes: Vec, + pub languages: Vec, + #[serde(default)] + pub year: Option, + #[serde(default)] + pub tmdb: Option, + #[serde(default)] + pub tvdb: Option, + #[serde(default)] + pub resolution: Option, + #[serde(default)] + pub quality: Option, + #[serde(default)] + pub codec: Option, + pub audio: Vec, + #[serde(default)] + pub group: Option, + #[serde(default)] + pub region: Option, + #[serde(default)] + pub container: Option, + #[serde(default)] + pub size: Option, + pub networks: Vec, + + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub extended: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub hardcoded: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub proper: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub repack: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub retail: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub remastered: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub unrated: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub uncensored: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub documentary: bool, + #[serde(default)] + pub episode_code: Option, + #[serde(default)] + pub date: Option, + + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub adult: bool, + #[serde(default)] + pub site: Option, + #[serde(default)] + pub bit_depth: Option, + #[serde(default)] + pub hdr: Vec, + #[serde(default)] + pub channels: Vec, + #[serde(default)] + pub volumes: Vec, + #[serde(default)] + pub edition: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub trash: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub upscaled: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub convert: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub commentary: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub subbed: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub dubbed: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub is_3d: bool, + + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub complete: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub ppv: bool, + #[serde(default)] + pub country: Option, + #[serde(default)] + pub bitrate: Option, + #[serde(default)] + pub extension: Option, +} \ No newline at end of file diff --git a/backend/src/ptt/parser.rs b/backend/src/ptt/parser.rs new file mode 100644 index 000000000..b425a9e13 --- /dev/null +++ b/backend/src/ptt/parser.rs @@ -0,0 +1,247 @@ +use std::collections::HashMap; +use fancy_regex::{Regex as FancyRegex, Match}; +use crate::ptt::constants::{BRACKETS, PTT_CONSTANTS}; +use crate::ptt::models::PttMetadata; + +#[derive(Debug, Clone)] +pub struct MatchInfo { + pub raw_match: String, + pub match_index: usize, + pub remove: bool, + pub skip_from_title: bool, +} + +pub struct ParseContext { + pub title: String, + pub result: PttMetadata, + pub matched: HashMap, +} + +pub type HandlerFn = Box Option + Send + Sync>; + +#[derive(Default)] +pub struct PttParser { + handlers: Vec<(String, HandlerFn)>, +} + +impl PttParser { + pub fn new() -> Self { + Self::default() + } + + pub fn add_handler_fn(&mut self, name: &str, handler: HandlerFn) { + self.handlers.push((name.to_string(), handler)); + } + + #[allow(clippy::needless_pass_by_value)] + pub fn add_handler( + &mut self, + name: &str, + regex: FancyRegex, + transformer: F, + setter: S, + options: HandlerOptions, + ) where + T: Clone + Send + Sync + 'static, + F: Fn(&str) -> T + Send + Sync + 'static, + S: Fn(&mut PttMetadata, T) + Send + Sync + 'static, + { + let name_owned = name.to_string(); + let name_clone = name.to_string(); + + let handler = Box::new(move |context: &mut ParseContext| -> Option { + if options.skip_if_already_found && context.matched.contains_key(&name_owned) { + return None; + } + + let match_result: Option = regex.find(&context.title).ok().flatten(); + + if let Some(m) = match_result { + let raw_match = m.as_str().to_string(); + let match_index = m.start(); + + let clean_match = if regex.captures_len() > 1 { + regex + .captures(&context.title) + .ok() + .flatten() + .and_then(|c| c.get(1).map(|g| g.as_str().to_string())) + .unwrap_or_else(|| raw_match.clone()) + } else { + raw_match.clone() + }; + + let before_match_opt = PTT_CONSTANTS.before_title_match_regex.find(&context.title); + let before_title_matched = if let Some(bm) = before_match_opt { + let bm_str = bm.as_str(); + if bm_str.len() > 2 && bm_str.starts_with('[') && bm_str.ends_with(']') { + let content = &bm_str[1..bm_str.len() - 1]; + content.contains(&raw_match) + } else { + false + } + } else { + false + }; + + let current_skip_from_title = before_title_matched || options.skip_from_title; + + let transformed_value = transformer(&clean_match); + + if options.skip_if_first { + let other_matches_exist = !context.matched.is_empty(); + if other_matches_exist { + let current_start = match_index; + let is_before_all = context + .matched + .values() + .all(|m| current_start < m.match_index); + if is_before_all { + return None; + } + } + } + + setter(&mut context.result, transformed_value); + + let info = MatchInfo { + raw_match, + match_index, + remove: options.remove, + skip_from_title: current_skip_from_title, + }; + context.matched.insert(name_owned.clone(), info.clone()); + + return Some(info); + } + None + }); + + self.handlers.push((name_clone, handler)); + } + + pub fn parse(&self, raw_title: &str, _translate_languages: bool) -> PttMetadata { + let mut context = ParseContext { + title: raw_title.to_string(), + result: PttMetadata::default(), + matched: HashMap::new(), + }; + + context.title = PTT_CONSTANTS.sub_pattern.replace_all(raw_title, " ").to_string(); + + let mut end_of_title = context.title.len(); + + for (_name, handler) in &self.handlers { + if let Some(match_info) = handler(&mut context) { + let match_index = match_info.match_index; + let raw_len = match_info.raw_match.len(); + + if match_info.remove && match_index + raw_len <= context.title.len() { + context + .title + .replace_range(match_index..match_index + raw_len, ""); + } + + if !match_info.skip_from_title && match_index > 1 && match_index < end_of_title { + end_of_title = match_index; + } + if match_info.remove + && match_info.skip_from_title + && match_index < end_of_title + && end_of_title >= raw_len + { + end_of_title -= raw_len; + } + } + } + + let mut result = context.result; + + result.seasons.sort_unstable(); + result.seasons.dedup(); + result.episodes.sort_unstable(); + result.episodes.dedup(); + result.languages.dedup(); + + let final_title = if end_of_title <= context.title.len() { + context.title[..end_of_title].to_string() + } else { + context.title + }; + + result.title = clean_title(&final_title); + + result + } +} + +pub fn clean_title(raw_title: &str) -> String { + let mut title = raw_title.replace('_', " "); + + title = PTT_CONSTANTS.movie_regex.replace_all(&title, "").to_string(); + title = PTT_CONSTANTS + .not_allowed_symbols_at_start_and_end + .replace_all(&title, "") + .to_string(); + title = PTT_CONSTANTS + .russian_cast_regex_fancy + .replace_all(&title, "") + .to_string(); + title = PTT_CONSTANTS.star_regex_1.replace_all(&title, "$1").to_string(); + title = PTT_CONSTANTS.star_regex_2.replace_all(&title, "$1").to_string(); + title = PTT_CONSTANTS.alt_titles_regex.replace_all(&title, "").to_string(); + title = PTT_CONSTANTS + .not_only_non_english_regex + .replace_all(&title, "$1") + .to_string(); + title = PTT_CONSTANTS + .remaining_not_allowed_symbols_at_start_and_end + .replace_all(&title, "") + .to_string(); + title = PTT_CONSTANTS.empty_brackets_regex.replace_all(&title, "").to_string(); + title = PTT_CONSTANTS.mp3_regex.replace_all(&title, "").to_string(); + title = PTT_CONSTANTS + .parantheses_without_content + .replace_all(&title, "") + .to_string(); + title = PTT_CONSTANTS + .special_char_spacing + .replace_all(&title, "") + .to_string(); + + for (open, close) in BRACKETS { + let open_count = title.matches(open).count(); + let close_count = title.matches(close).count(); + if open_count != close_count { + title = title.replace(open, "").replace(close, ""); + } + } + + if !title.trim().contains(' ') && title.contains('.') { + title = PTT_CONSTANTS.dot.replace_all(&title, " ").to_string(); + } + + title = PTT_CONSTANTS.redundant_symbols_at_end.replace_all(&title, "").to_string(); + title = PTT_CONSTANTS.spacing_regex.replace_all(&title, " ").trim().to_string(); + + title +} + +#[allow(clippy::struct_excessive_bools)] +pub struct HandlerOptions { + pub skip_if_already_found: bool, + pub skip_from_title: bool, + pub skip_if_first: bool, + pub remove: bool, +} + +impl Default for HandlerOptions { + fn default() -> Self { + Self { + skip_if_already_found: true, + skip_from_title: false, + skip_if_first: false, + remove: false, + } + } +} diff --git a/backend/src/ptt/transformers.rs b/backend/src/ptt/transformers.rs new file mode 100644 index 000000000..99a84a46e --- /dev/null +++ b/backend/src/ptt/transformers.rs @@ -0,0 +1,120 @@ +use chrono::NaiveDate; +use crate::ptt::constants::PTT_CONSTANTS; + +pub fn none(input: &str) -> String { + input.to_string() +} + +pub fn value(val: &str) -> String { + val.to_string() +} + +pub fn boolean(_: &str) -> bool { + true +} +pub fn uinteger(input: &str) -> Option { + input.parse::().ok() +} + +pub fn first_uinteger(input: &str) -> Option { + PTT_CONSTANTS.integer.find(input).and_then(|m| m.as_str().parse::().ok()) +} + +pub fn lowercase(input: &str) -> String { + input.to_lowercase() +} + +pub fn uppercase(input: &str) -> String { + input.to_uppercase() +} + +pub fn convert_months(date_str: &str) -> String { + let mut result = date_str.to_string(); + for (re, short) in &PTT_CONSTANTS.months { + result = re.replace_all(&result, *short).to_string(); + } + result +} + +pub fn date(input: &str, formats: &[&str]) -> Option { + let sanitized = PTT_CONSTANTS.word + .replace_all(input, " ") + .trim() + .to_string(); + let sanitized = convert_months(&sanitized); + + for fmt in formats { + if let Ok(dt) = NaiveDate::parse_from_str(&sanitized, fmt) { + return Some(dt.format("%Y-%m-%d").to_string()); + } + } + None +} + +macro_rules! range_func { + ($name:ident, $t:ty) => { + pub fn $name(input: &str) -> Option> { + let numbers: Vec<$t> = PTT_CONSTANTS.integer + .find_iter(input) + .filter_map(|mat| mat.as_str().parse().ok()) + .collect(); + + if numbers.len() == 2 && numbers[0] < numbers[1] { + return Some((numbers[0]..=numbers[1]).collect()); + } + match numbers.len() { + len if len > 2 => { + if numbers.windows(2).all(|w| w[0] + 1 == w[1]) { + Some(numbers) + } else { + None + } + } + 1 => Some(numbers), + _ => None, + } + } + }; +} + +range_func!(range_i32, i32); +range_func!(range_u32, u32); + +// +// pub fn range_x_of_y_func(input: &str) -> Option> { +// let numbers: Vec = PTT_CONSTANTS.integer +// .find_iter(input) +// .filter_map(|mat| mat.as_str().parse().ok()) +// .collect(); +// +// if numbers.len() != 1 { +// return None; +// } +// Some((1..=numbers[0]).collect()) +// } + +pub fn transform_resolution(input: &str) -> String { + let lower = input.to_lowercase(); + if lower.contains("2160") || lower.contains("4k") { + return "2160p".to_string(); + } + if lower.contains("1440") || lower.contains("2k") { + return "1440p".to_string(); + } + if lower.contains("1080") { + return "1080p".to_string(); + } + if lower.contains("720") { + return "720p".to_string(); + } + if lower.contains("480") { + return "480p".to_string(); + } + if lower.contains("360") { + return "360p".to_string(); + } + if lower.contains("240") { + return "240p".to_string(); + } + input.to_string() +} \ No newline at end of file diff --git a/backend/src/repository/bplustree.rs b/backend/src/repository/bplustree.rs index 5b15f4eef..0756a90cd 100644 --- a/backend/src/repository/bplustree.rs +++ b/backend/src/repository/bplustree.rs @@ -10,7 +10,7 @@ use std::marker::PhantomData; use std::mem::size_of; use std::path::Path; use tempfile::NamedTempFile; -use crate::utils::{bincode_deserialize, bincode_serialize}; +use crate::utils::{binary_deserialize, binary_serialize}; const BLOCK_SIZE: usize = 4096; const BINCODE_OVERHEAD: usize = 8; @@ -288,7 +288,7 @@ where let mut write_pos = FLAG_SIZE; // ---- Write keys (length + bytes) into the first block ---- - let keys_encoded = bincode_serialize(&self.keys)?; + let keys_encoded = binary_serialize(&self.keys)?; let keys_len = keys_encoded.len(); buffer_slice[write_pos..write_pos + LEN_SIZE] .copy_from_slice(&u32::try_from(keys_len).map_err(to_io_error)?.to_le_bytes()); @@ -304,7 +304,7 @@ where // ---- Leaf values (optional) ---- if self.is_leaf { // Encode values and decide compression exactly like the old layout - let values_encoded = bincode_serialize(&self.values)?; + let values_encoded = binary_serialize(&self.values)?; let use_compression = values_encoded.len() + write_pos >= BLOCK_SIZE; // Compression flag @@ -369,7 +369,7 @@ where pointer_vec.push(current_offset); current_offset = child.serialize_to_block(file, buffer, current_offset)?; } - let pointer_encoded = bincode_serialize(&pointer_vec)?; + let pointer_encoded = binary_serialize(&pointer_vec)?; let pointer_len = u32::try_from(pointer_encoded.len()).map_err(to_io_error)?; file.seek(SeekFrom::Start(pointer_offset))?; file.write_all(&pointer_len.to_le_bytes())?; @@ -396,7 +396,7 @@ where // ---- Keys ---- let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize; read_pos += LEN_SIZE; - let keys: Vec = bincode_deserialize(&buffer[read_pos..read_pos + keys_length])?; + let keys: Vec = binary_deserialize(&buffer[read_pos..read_pos + keys_length])?; read_pos += keys_length; // ---- Values for leaf nodes ---- @@ -429,7 +429,7 @@ where content_bytes }; - bincode_deserialize(&values_bytes)? + binary_deserialize(&values_bytes)? } else { Vec::new() }; @@ -440,7 +440,7 @@ where } else { let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize; read_pos += LEN_SIZE; - let pointers: Vec = bincode_deserialize(&buffer[read_pos..read_pos + pointers_length])?; + let pointers: Vec = binary_deserialize(&buffer[read_pos..read_pos + pointers_length])?; if nested { let mut nodes = Vec::with_capacity(pointers.len()); for &ptr in &pointers { diff --git a/backend/src/repository/indexed_document.rs b/backend/src/repository/indexed_document.rs index 9323e12d7..43b572c2f 100644 --- a/backend/src/repository/indexed_document.rs +++ b/backend/src/repository/indexed_document.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use tempfile::NamedTempFile; use shared::error::{str_to_io_error, to_io_error}; use crate::utils; -use crate::utils::{bincode_deserialize, bincode_serialize, IO_BUFFER_SIZE}; +use crate::utils::{binary_deserialize, binary_serialize, IO_BUFFER_SIZE}; const BLOCK_SIZE: usize = 4096; const LEN_SIZE: usize = 4; @@ -169,8 +169,8 @@ where self.main_file.seek(SeekFrom::Start(u64::from(offset)))?; let old_size = IndexedDocument::read_content_size(&mut self.main_file)?; - // Prepare new payload (bincode utils, compatible with project) - let encoded = bincode_serialize(doc) + // Prepare new payload (binary utils, compatible with project) + let encoded = binary_serialize(doc) .map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?; let new_size: SizeType = SizeType::try_from(encoded.len() as u64) .map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?; @@ -206,7 +206,7 @@ where // New entry -> append self.main_file.seek(SeekFrom::End(0))?; // Determine size and write it, then write payload - let encoded = bincode_serialize(doc) + let encoded = binary_serialize(doc) .map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?; let new_size: SizeType = SizeType::try_from(encoded.len() as u64) .map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?; @@ -237,7 +237,6 @@ where } } - //////////////////////////////////////////////////////// /// /// `IndexedDocumentReader` @@ -282,7 +281,7 @@ where let buf_size = IndexedDocument::read_content_size(&mut self.main_file)?; self.buffer.resize(buf_size, 0u8); self.main_file.read_exact(&mut self.buffer[..buf_size])?; - if let Ok(item) = bincode_deserialize::(&self.buffer[..buf_size]) { + if let Ok(item) = binary_deserialize::(&self.buffer[..buf_size]) { return Ok(item); } } @@ -369,7 +368,7 @@ where // read content self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?; // deserialize buffer - match bincode_deserialize::(&self.t_buffer[0..buf_size]) { + match binary_deserialize::(&self.t_buffer[0..buf_size]) { Ok(value) => Ok(Some((value, has_next))), Err(err) => { self.failed = true; @@ -417,7 +416,7 @@ impl IndexedDocumentDirectAccess { let buf_size = IndexedDocument::read_content_size(&mut main_file)?; let mut buffer: Vec = vec![0; buf_size]; main_file.read_exact(&mut buffer)?; - if let Ok(item) = bincode_deserialize::(&buffer) { + if let Ok(item) = binary_deserialize::(&buffer) { return Ok(item); } } diff --git a/backend/src/repository/mod.rs b/backend/src/repository/mod.rs index 1e210a2bd..4cb7f1af0 100644 --- a/backend/src/repository/mod.rs +++ b/backend/src/repository/mod.rs @@ -12,4 +12,5 @@ pub mod m3u_playlist_iterator; pub mod xtream_playlist_iterator; pub mod user_repository; pub mod storage_const; +mod playlist_scratch; diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index a199ebd76..67953c683 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -1,21 +1,21 @@ use crate::api::model::{AppState, PlaylistM3uStorage, PlaylistStorage, PlaylistStorageState, PlaylistXtreamStorage}; -use crate::model::{AppConfig, ConfigTarget, TargetOutput}; -use crate::model::{Epg}; +use crate::model::{AppConfig, ConfigInput, ConfigTarget, TargetOutput}; +use crate::model::Epg; use crate::processing::processor::playlist::apply_filter_to_playlist; use crate::repository::bplustree::BPlusTree; use crate::repository::epg_repository::epg_write; use crate::repository::indexed_document::IndexedDocumentIterator; use crate::repository::m3u_repository::{m3u_get_file_paths, m3u_write_playlist}; -use crate::repository::storage::{ensure_target_storage_path, get_target_id_mapping_file, get_target_storage_path}; +use crate::repository::storage::{ensure_target_storage_path, get_input_storage_path, get_target_id_mapping_file, get_target_storage_path}; use crate::repository::strm_repository::write_strm_playlist; use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord}; -use crate::repository::xtream_repository::{xtream_get_file_paths, xtream_get_storage_path, xtream_write_playlist}; +use crate::repository::xtream_repository::{persist_input_xtream_playlist, xtream_get_file_paths, xtream_get_storage_path, xtream_write_playlist}; use crate::utils; use log::info; use shared::create_tuliprox_error; use shared::error::TuliproxError; use shared::error::{info_err, TuliproxErrorKind}; -use shared::model::{M3uPlaylistItem, PlaylistGroup, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem}; +use shared::model::{InputType, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem}; use shared::utils::{is_dash_url, is_hls_url}; use std::collections::HashMap; use std::path::Path; @@ -26,7 +26,12 @@ struct LocalEpisodeKey { virtual_id: u32, } -pub async fn persist_playlist(app_config: &AppConfig, playlist: &mut [PlaylistGroup], epg: Option<&Epg>, +pub struct ProviderEpisodeKey { + pub(crate) provider_id: u32, + pub(crate) virtual_id: u32, +} + +pub async fn persist_playlist(app_config: &Arc, playlist: &mut [PlaylistGroup], epg: Option<&Epg>, target: &ConfigTarget, playlist_state: Option<&Arc>) -> Result<(), Vec> { let mut errors = vec![]; let config = &app_config.config.load(); @@ -38,6 +43,7 @@ pub async fn persist_playlist(app_config: &AppConfig, playlist: &mut [PlaylistGr let (mut target_id_mapping, file_lock) = get_target_id_mapping(app_config, &target_path).await; let mut local_library_series = HashMap::>::new(); + let mut provider_series = HashMap::>::new(); // Virtual IDs assignment for group in playlist.iter_mut() { @@ -62,11 +68,16 @@ pub async fn persist_playlist(app_config: &AppConfig, playlist: &mut [PlaylistGr let item_type = header.item_type; header.virtual_id = target_id_mapping.get_and_update_virtual_id(uuid, provider_id, item_type, 0); - assign_local_series_info_episode_key(&mut local_library_series, header, item_type); + if item_type == PlaylistItemType::LocalSeries { + assign_local_series_info_episode_key(&mut local_library_series, header, item_type); + } + if item_type == PlaylistItemType::Series { + assign_provider_series_info_episode_key(&mut provider_series, header, item_type); + } } } - rewrite_local_series_info_episode_virtual_id(playlist, &mut local_library_series); + rewrite_series_info_episode_virtual_id(playlist, &local_library_series, &provider_series); for output in &target.output { let mut filtered = match output { @@ -142,32 +153,32 @@ fn assign_local_series_info_episode_key(local_library_series: &mut HashMap>) { - // assign local series virtual ids - for group in playlist.iter_mut() { - for channel in &mut group.channels { - let header = &mut channel.header; - if header.item_type == PlaylistItemType::LocalSeriesInfo { - if let Some(episode_keys) = local_library_series.get(&header.id) { - if let Some(stream_props) = header.additional_properties.as_mut() { - match stream_props { - StreamProperties::Live(_) - | StreamProperties::Video(_) - | StreamProperties::Episode(_) => {} - StreamProperties::Series(series) => { - if let Some(episodes) = - series.details.as_mut().and_then(|d| d.episodes.as_mut()) - { - for episode in episodes.iter_mut() { - for episode_key in episode_keys { - if episode.direct_source == episode_key.path { - episode.id = episode_key.virtual_id; - break; - } - } - } - } - } +fn assign_provider_series_info_episode_key(provider_series: &mut HashMap>, header: &mut PlaylistItemHeader, item_type: PlaylistItemType) { + // we need to rewrite local series info with the new virtual ids + if item_type == PlaylistItemType::Series { + provider_series + .entry(header.parent_code.clone()) + .or_default() + .push(ProviderEpisodeKey { + provider_id: header.get_provider_id().unwrap_or_default(), + virtual_id: header.virtual_id, + }); + } +} + +#[allow(clippy::implicit_hasher)] +fn rewrite_local_series_info_episode_virtual_id(pli: &mut PlaylistItem, local_library_series: &HashMap>) { + let header = &mut pli.header; + if let Some(episode_keys) = local_library_series.get(&header.id) { + if let Some(StreamProperties::Series(series)) = header.additional_properties.as_mut() { + if let Some(episodes) = + series.details.as_mut().and_then(|d| d.episodes.as_mut()) + { + for episode in episodes.iter_mut() { + for episode_key in episode_keys { + if episode.direct_source == episode_key.path { + episode.id = episode_key.virtual_id; + break; } } } @@ -176,6 +187,46 @@ fn rewrite_local_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup], } } +#[allow(clippy::implicit_hasher)] +pub fn rewrite_provider_series_info_episode_virtual_id

(pli: &mut P, provider_series: &HashMap>) +where P: PlaylistEntry +{ + if let Some(episode_keys) = provider_series.get(&pli.get_uuid().to_string()) { + if let Some(StreamProperties::Series(series)) = pli.get_additional_properties_mut() { + if let Some(episodes) = + series.details.as_mut().and_then(|d| d.episodes.as_mut()) + { + for episode in episodes.iter_mut() { + for episode_key in episode_keys { + if episode.id == episode_key.provider_id { + episode.id = episode_key.virtual_id; + break; + } + } + } + } + } + } +} + +fn rewrite_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup], + local_library_series: &HashMap>, + provider_series: &HashMap>) { + if local_library_series.is_empty() && provider_series.is_empty() { + return; + } + for group in playlist.iter_mut() { + for channel in &mut group.channels { + let item_type = channel.header.item_type; + if item_type == PlaylistItemType::SeriesInfo { + rewrite_provider_series_info_episode_virtual_id(channel, provider_series); + } else if item_type == PlaylistItemType::LocalSeriesInfo { + rewrite_local_series_info_episode_virtual_id(channel, local_library_series); + } + } + } +} + pub async fn get_target_id_mapping(cfg: &AppConfig, target_path: &Path) -> (TargetIdMapping, utils::FileWriteGuard) { let target_id_mapping_file = get_target_id_mapping_file(target_path); let file_lock = cfg.file_locks.write_lock(&target_id_mapping_file).await; @@ -186,12 +237,8 @@ pub async fn get_target_id_mapping(cfg: &AppConfig, target_path: &Path) -> (Targ async fn load_target_id_mapping_as_tree(app_config: &AppConfig, target_path: &Path, target: &ConfigTarget) -> Result, TuliproxError> { let target_id_mapping_file = get_target_id_mapping_file(target_path); let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; - BPlusTree::::load(&target_id_mapping_file).map_err(|err| - create_tuliprox_error!( - TuliproxErrorKind::Info, - "Could not find path for target {} err:{err}", &target.name - )) + create_tuliprox_error!(TuliproxErrorKind::Info, "Could not find path for target {} err:{err}", &target.name)) } async fn load_xtream_playlist_as_tree(app_config: &AppConfig, storage_path: &Path, cluster: XtreamCluster) -> BPlusTree { @@ -281,3 +328,23 @@ pub async fn load_target_into_memory_cache(app_state: &AppState, target: &Arc, input: &Arc, mut playlist: Vec) -> (Vec, Option) { + playlist.iter_mut().for_each(PlaylistGroup::on_load); + + let (result, err) = match input.input_type { + InputType::Xtream | InputType::XtreamBatch => { + let working_dir = &app_config.config.load().working_dir; + let storage_path = match get_input_storage_path(&input.name, working_dir) { + Ok(storage_path) => storage_path, + Err(err) => { + return (playlist, Some(create_tuliprox_error!( TuliproxErrorKind::Info, "Error creating input storage directory for input '{}' failed: {err}", input.name))); + } + }; + persist_input_xtream_playlist(app_config, &storage_path, playlist).await + } + _ => (playlist, None), + }; + + (result, err) +} \ No newline at end of file diff --git a/backend/src/repository/playlist_scratch.rs b/backend/src/repository/playlist_scratch.rs new file mode 100644 index 000000000..3e6386cd6 --- /dev/null +++ b/backend/src/repository/playlist_scratch.rs @@ -0,0 +1,97 @@ +use indexmap::IndexMap; +use shared::model::XtreamCluster; + +pub(in crate::repository) trait WithCapacity { + fn with_capacity(capacity: usize) -> Self; +} + +impl WithCapacity for Vec { + fn with_capacity(capacity: usize) -> Self { + Vec::with_capacity(capacity) + } +} + +impl WithCapacity for IndexMap { + fn with_capacity(capacity: usize) -> Self { + IndexMap::with_capacity(capacity) + } +} + +pub(in crate::repository) trait IsEmpty { + fn is_empty(&self) -> bool; +} + + +impl IsEmpty for Vec { + fn is_empty(&self) -> bool { + Vec::is_empty(self) + } +} + +impl IsEmpty for IndexMap { + fn is_empty(&self) -> bool { + IndexMap::is_empty(self) + } +} + +pub struct PlaylistScratch { + live: C, + vods: C, + series: C, +} + +impl PlaylistScratch { + pub fn new(capacity: usize) -> Self { + Self { + live: C::with_capacity(capacity), + vods: C::with_capacity(capacity), + series: C::with_capacity(capacity), + } + } + + pub fn get_mut(&mut self, cluster: XtreamCluster) -> &mut C { + match cluster { + XtreamCluster::Live => &mut self.live, + XtreamCluster::Video => &mut self.vods, + XtreamCluster::Series => &mut self.series, + } + } + + pub fn get(&self, cluster: XtreamCluster) -> &C { + match cluster { + XtreamCluster::Live => &self.live, + XtreamCluster::Video => &self.vods, + XtreamCluster::Series => &self.series, + } + } + + pub fn set(&mut self, cluster: XtreamCluster, content: C) { + match cluster { + XtreamCluster::Live => self.live = content, + XtreamCluster::Video => self.vods = content, + XtreamCluster::Series => self.series = content, + } + } +} + +impl PlaylistScratch { + pub fn take(&mut self, cluster: XtreamCluster) -> C { + match cluster { + XtreamCluster::Live => std::mem::take(&mut self.live), + XtreamCluster::Video => std::mem::take(&mut self.vods), + XtreamCluster::Series => std::mem::take(&mut self.series), + } + } +} + +impl PlaylistScratch { + + pub fn is_empty(&self, cluster: XtreamCluster) -> bool { + match cluster { + XtreamCluster::Live => self.live.is_empty(), + XtreamCluster::Video => self.vods.is_empty(), + XtreamCluster::Series => self.series.is_empty(), + } + } +} + diff --git a/backend/src/repository/storage_const.rs b/backend/src/repository/storage_const.rs index 000eeb816..e80c3168c 100644 --- a/backend/src/repository/storage_const.rs +++ b/backend/src/repository/storage_const.rs @@ -27,5 +27,4 @@ pub(in crate::repository) const FILE_SERIES_INFO_RECORD: &str = "series_info_rec pub(in crate::repository) const FILE_SERIES_EPISODE_RECORD: &str = "series_episode_record"; // pub(in crate::repository) const FILE_SERIES: &str = "series"; pub(in crate::repository) const PATH_XTREAM: &str = "xtream"; -pub(in crate::repository) const INFO_REWRITE_FIELDS: &[&str] = &["cover_big", "cover", "cover_tmdb", "movie_image", "tmdb_url", "overview", "kinopoisk_url"]; diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 0549fdf4f..933ad3af7 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -2,12 +2,9 @@ use crate::model::MediaQuality; use crate::model::{ApiProxyServerInfo, AppConfig, ProxyUserCredentials}; use crate::model::{ConfigTarget, StrmTargetOutput}; -use crate::repository::bplustree::BPlusTree; -use crate::repository::storage::{ensure_target_storage_path, get_input_storage_path}; +use crate::repository::storage::{ensure_target_storage_path}; use crate::repository::storage_const; -use crate::repository::xtream_repository::{xtream_get_record_file_path, InputVodInfoRecord}; -use crate::utils; -use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, FileReadGuard, IO_BUFFER_SIZE}; +use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, IO_BUFFER_SIZE}; use chrono::Datelike; use filetime::{set_file_times, FileTime}; use log::{error, trace}; @@ -15,7 +12,7 @@ use regex::Regex; use serde::Serialize; use shared::error::{create_tuliprox_error_result, info_err}; use shared::error::{TuliproxError, TuliproxErrorKind}; -use shared::model::{ClusterFlags, EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType}; +use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType}; use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; @@ -125,91 +122,6 @@ fn style_rename_year<'a>( (std::borrow::Cow::Owned(new_name), smallest_year) } -#[derive(Clone)] -enum InputTmdbIndexTree { - Video(BPlusTree), - Series(BPlusTree), -} - -#[derive(Clone)] -enum InputTmdbIndexValue { - Video(InputVodInfoRecord), - Series(EpisodeStreamProperties), -} - -type InputTmdbIndexMap = HashMap>; -async fn get_tmdb_value( - cfg: &AppConfig, - provider_id: Option, - input_name: &str, - input_indexes: &mut InputTmdbIndexMap, - item_type: PlaylistItemType, -) -> Option { - // the tmdb_ids are stored inside record files for xtream input. - // we load this record files on request for each input and item_type. - let pid = provider_id?; - match input_indexes.entry(input_name.to_string()) { - std::collections::hash_map::Entry::Occupied(entry) => { - if let Some((_, tree_value)) = entry.get() { - match tree_value { - InputTmdbIndexTree::Video(tree) => tree - .query(&pid) - .map(|vod_record| InputTmdbIndexValue::Video(vod_record.clone())), - InputTmdbIndexTree::Series(tree) => tree - .query(&pid) - .map(|episode| InputTmdbIndexValue::Series(episode.clone())), - } - } else { - None - } - } - std::collections::hash_map::Entry::Vacant(entry) => { - if let Ok(Some(tmdb_path)) = get_input_storage_path(input_name, &cfg.config.load().working_dir) - .map(|storage_path| xtream_get_record_file_path(&storage_path, item_type)) - { - { - let file_lock = cfg.file_locks.read_lock(&tmdb_path).await; - match item_type { - PlaylistItemType::Series - | PlaylistItemType::LocalSeries => { - if let Ok(tree) = - BPlusTree::::load(&tmdb_path) - { - let tmdb_id = tree.query(&pid).map(|episode| { - InputTmdbIndexValue::Series(episode.clone()) - }); - entry.insert(Some(( - file_lock, - InputTmdbIndexTree::Series(tree), - ))); - return tmdb_id; - } - } - PlaylistItemType::Video - | PlaylistItemType::LocalVideo => { - if let Ok(tree) = - BPlusTree::::load(&tmdb_path) - { - let tmdb_id = tree.query(&pid).map(|vod_record| { - InputTmdbIndexValue::Video(vod_record.clone()) - }); - entry.insert(Some(( - file_lock, - InputTmdbIndexTree::Video(tree), - ))); - return tmdb_id; - } - } - _ => {} - } - }; - } - entry.insert(None); - None - } - } -} - pub fn strm_get_file_paths(file_prefix: &str, target_path: &Path) -> PathBuf { target_path.join(PathBuf::from(format!("{file_prefix}_{}.{}", storage_const::FILE_STRM, storage_const::FILE_SUFFIX_DB))) } @@ -647,29 +559,17 @@ fn format_for_jellyfin( /// Generates style-compliant directory and file names by dispatching /// the call to a dedicated formatting function for the respective style. -async fn style_based_rename( - cfg: &AppConfig, +fn style_based_rename( strm_item_info: &StrmItemInfo, - input_tmdb_indexes: &mut InputTmdbIndexMap, - style: &StrmExportStyle, + tmdb: Option, + style: StrmExportStyle, underscore_whitespace: bool, flat: bool, ) -> (PathBuf, String) { let separator = if underscore_whitespace { "_" } else { " " }; - let tmdb_id_val = get_tmdb_value( - cfg, - strm_item_info.provider_id, - strm_item_info.input_name.as_str(), - input_tmdb_indexes, - strm_item_info.item_type, - ).await; - let tmdb_id = match tmdb_id_val { - Some(InputTmdbIndexValue::Video(r)) if r.tmdb_id != 0 => r.tmdb_id, - Some(InputTmdbIndexValue::Series(e)) if e.tmdb.is_some_and(|e| e != 0) => e.tmdb.unwrap_or_default(), - _ => strm_item_info.tmdb_id.unwrap_or(0), - }; + let tmdb_id = tmdb.or(strm_item_info.tmdb_id).unwrap_or(0); // Dispatch the call to the responsible function based on the style. match style { @@ -680,8 +580,8 @@ async fn style_based_rename( } } -async fn prepare_strm_files( - cfg: &AppConfig, +fn prepare_strm_files( + _app_config: &AppConfig, new_playlist: &mut [PlaylistGroup], _root_path: &Path, strm_target_output: &StrmTargetOutput, @@ -694,7 +594,6 @@ async fn prepare_strm_files( let mut all_filenames = HashSet::with_capacity(channel_count); // contains only collision filenames let mut collisions: HashSet> = HashSet::new(); - let mut input_tmdb_indexes: InputTmdbIndexMap = HashMap::with_capacity(channel_count); let mut result = Vec::with_capacity(channel_count); // first we create the names to identify name collisions @@ -703,13 +602,12 @@ async fn prepare_strm_files( let strm_item_info = extract_item_info(pli); let (dir_path, strm_file_name) = style_based_rename( - cfg, &strm_item_info, - &mut input_tmdb_indexes, - &strm_target_output.style, + pli.get_tmdb_id(), + strm_target_output.style, strm_target_output.underscore_whitespace, strm_target_output.flat, - ).await; + ); // Conditionally generate the quality string based on the new config flag let separator = if strm_target_output.underscore_whitespace { "_" } else { " " }; @@ -803,7 +701,7 @@ pub async fn write_strm_playlist( } let config = app_config.config.load(); - let Some(root_path) = utils::get_file_path( + let Some(root_path) = crate::utils::get_file_path( &config.working_dir, Some(std::path::PathBuf::from(&target_output.directory)), ) else { @@ -839,7 +737,7 @@ pub async fn write_strm_playlist( new_playlist, &root_path, target_output, - ).await; + ); for strm_file in strm_files { // file paths let output_path = truncate_filename(&root_path.join(&strm_file.dir_path), 255); diff --git a/backend/src/repository/user_repository.rs b/backend/src/repository/user_repository.rs index 97fc07295..fc88f2d6c 100644 --- a/backend/src/repository/user_repository.rs +++ b/backend/src/repository/user_repository.rs @@ -494,7 +494,6 @@ mod tests { sources_file_path: String::new(), mapping_file_path: None, api_proxy_file_path: String::new(), - library_file_path: String::new(), custom_stream_response_path: None, }))), file_locks: Arc::new(FileLockManager::default()), diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index 7957a3774..ea806fb51 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -5,7 +5,6 @@ use crate::repository::user_repository::user_get_bouquet_filter; use crate::repository::xtream_repository::{xtream_get_file_paths, xtream_get_storage_path}; use crate::utils::FileReadGuard; use log::error; -use serde_json::Value; use shared::error::info_err; use shared::error::TuliproxError; use shared::model::{PlaylistItemType, TargetType, XtreamCluster, XtreamMappingOptions, XtreamPlaylistItem}; @@ -148,14 +147,10 @@ impl XtreamPlaylistJsonIterator { } } -pub fn to_doc(pli: &XtreamPlaylistItem, options: &XtreamMappingOptions) -> Value { - pli.to_document(options) -} - impl Iterator for XtreamPlaylistJsonIterator { type Item = (String, bool); fn next(&mut self) -> Option { - self.inner.get_next().map(|(pli, has_next)| (to_doc(&pli, &self.inner.options).to_string(), has_next)) + self.inner.get_next().map(|(pli, has_next)| (pli.to_document(&self.inner.options).to_string(), has_next)) } } diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index f50ebc063..9b7c122ee 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -1,26 +1,27 @@ use crate::api::model::AppState; -use crate::model::{normalize_release_date, xtream_mapping_option_from_target_options}; -use crate::model::{rewrite_doc_urls, PlaylistXtreamCategory}; +use crate::model::PlaylistXtreamCategory; use crate::model::{AppConfig, ProxyUserCredentials}; -use crate::model::{Config, ConfigInput, ConfigTarget, XtreamTargetOutput}; +use crate::model::{Config, ConfigInput, ConfigTarget}; use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate}; use crate::repository::indexed_document::{IndexedDocumentDirectAccess, IndexedDocumentGarbageCollector, IndexedDocumentIterator, IndexedDocumentWriter}; -use crate::repository::playlist_repository::get_target_id_mapping; +use crate::repository::playlist_scratch::PlaylistScratch; use crate::repository::storage::{get_input_storage_path, get_target_id_mapping_file, get_target_storage_path}; use crate::repository::storage_const; use crate::repository::target_id_mapping::VirtualIdRecord; use crate::repository::xtream_playlist_iterator::XtreamPlaylistJsonIterator; -use crate::utils::{async_file_reader, async_open_readonly_file, file_reader}; use crate::utils::json_write_documents_to_file; -use crate::utils::{bincode_deserialize, FileReadGuard}; +use crate::utils::{async_file_reader, async_open_readonly_file, file_reader}; +use crate::utils::{FileReadGuard}; use bytes::Bytes; use futures::{stream, Stream, StreamExt}; +use indexmap::IndexMap; use log::error; use serde::Serialize; -use serde_json::{json, Map, Value}; +use serde_json::{json, Value}; use shared::error::{create_tuliprox_error, create_tuliprox_error_result, info_err, notify_err, str_to_io_error, to_io_error, TuliproxError, TuliproxErrorKind}; -use shared::model::{EpisodeStreamProperties, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{generate_playlist_uuid, get_u32_from_serde_value, hex_encode}; +use shared::model::xtream_const::XTREAM_CLUSTER; +use shared::model::{LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemType, SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}; +use shared::utils::get_u32_from_serde_value; use std::collections::HashMap; use std::fs; use std::fs::File; @@ -49,18 +50,35 @@ macro_rules! try_option_ok { } }; } +// +// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] +// pub struct InputVodInfoRecord { +// pub(crate) tmdb_id: u32, +// pub(crate) ts: u64, +// pub(crate) release_date: Option, +// } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] -pub struct InputVodInfoRecord { - pub(crate) tmdb_id: u32, - pub(crate) ts: u64, - pub(crate) release_date: Option, -} - +#[inline] fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } +#[inline] +fn get_live_cat_collection_path(path: &Path) -> PathBuf { + get_collection_path(path, storage_const::COL_CAT_LIVE) +} + +#[inline] +fn get_vod_cat_collection_path(path: &Path) -> PathBuf { + get_collection_path(path, storage_const::COL_CAT_VOD) +} + +#[inline] +fn get_series_cat_collection_path(path: &Path) -> PathBuf { + get_collection_path(path, storage_const::COL_CAT_SERIES) +} + + fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { if let Some(path) = xtream_get_storage_path(cfg, target_name) { if std::fs::create_dir_all(&path).is_err() { @@ -105,7 +123,7 @@ pub fn xtream_get_record_file_path(storage_path: &Path, item_type: PlaylistItemT } } async fn write_playlists_to_file( - cfg: &AppConfig, + app_config: &Arc, storage_path: &Path, collections: Vec<(XtreamCluster, &[&mut PlaylistItem])>, ) -> Result<(), TuliproxError> { @@ -115,7 +133,7 @@ async fn write_playlists_to_file( } let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); { - let _file_lock = cfg.file_locks.write_lock(&xtream_path).await; + let _file_lock = app_config.file_locks.write_lock(&xtream_path).await; match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { Ok(mut writer) => { for item in playlist { @@ -134,6 +152,59 @@ async fn write_playlists_to_file( Ok(()) } +async fn write_playlists_to_file_2( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + playlist: &[PlaylistItem], +) -> Result<(), TuliproxError> { + if !playlist.is_empty() { + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); + let _file_lock = app_config.file_locks.write_lock(&xtream_path).await; + match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { + Ok(mut writer) => { + for item in playlist { + let xtream = item.to_xtream(); + match writer.write_doc(item.header.virtual_id, &xtream) { + Ok(()) => {} + Err(err) => return Err(cant_write_result!(&xtream_path, err)), + } + } + writer.store().map_err(|err| cant_write_result!(&xtream_path, err))?; + } + Err(err) => return Err(cant_write_result!(&xtream_path, err)), + } + } + Ok(()) +} + +pub async fn write_playlist_item_to_file( + app_config: &Arc, + target_name: &str, + pli: &XtreamPlaylistItem, +) -> Result<(), TuliproxError> { + let storage_path = { + let config = app_config.config.load(); + ensure_xtream_storage_path(&config, target_name)? + }; + let (xtream_path, idx_path) = xtream_get_file_paths(&storage_path, pli.xtream_cluster); + { + let _file_lock = app_config.file_locks.write_lock(&xtream_path).await; + match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { + Ok(mut writer) => { + match writer.write_doc(pli.virtual_id, pli) { + Ok(()) => {} + Err(err) => return Err(cant_write_result!(&xtream_path, err)), + } + writer.store().map_err(|err| cant_write_result!(&xtream_path, err))?; + } + Err(err) => return Err(cant_write_result!(&xtream_path, err)), + } + } + Ok(()) +} + + fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Option { if let Some(value) = map.get(key) { if let Some(result) = value.as_str() { @@ -199,10 +270,10 @@ pub fn xtream_get_file_paths(storage_path: &Path, cluster: XtreamCluster) -> (Pa // xtream_get_file_paths_for_name(storage_path, storage_const::FILE_SERIES) // } -async fn xtream_garbage_collect(config: &AppConfig, target_name: &str) -> std::io::Result<()> { +async fn xtream_garbage_collect(app_cfg: &Arc, target_name: &str) -> std::io::Result<()> { // Garbage collect series let storage_path = { - let cfg = config.config.load(); + let cfg = app_cfg.config.load(); try_option_ok!(xtream_get_storage_path(&cfg, target_name)) }; let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths( @@ -210,7 +281,7 @@ async fn xtream_garbage_collect(config: &AppConfig, target_name: &str) -> std::i XtreamCluster::Series )); { - let _file_lock = config.file_locks.write_lock(&info_path).await; + let _file_lock = app_cfg.file_locks.write_lock(&info_path).await; IndexedDocumentGarbageCollector::::new(info_path.clone(), idx_path)?.garbage_collect()?; } Ok(()) @@ -224,12 +295,12 @@ struct CategoryEntry { } pub async fn xtream_write_playlist( - cfg: &AppConfig, + app_cfg: &Arc, target: &ConfigTarget, playlist: &mut [PlaylistGroup], ) -> Result<(), TuliproxError> { let path = { - let config = cfg.config.load(); + let config = app_cfg.config.load(); ensure_xtream_storage_path(&config, target.name.as_str())? }; let mut errors = Vec::new(); @@ -237,8 +308,8 @@ pub async fn xtream_write_playlist( let mut cat_series_col = Vec::with_capacity(1_000); let mut cat_vod_col = Vec::with_capacity(1_000); let mut live_col = Vec::with_capacity(50_000); - let mut series_col = Vec::with_capacity(10_000); - let mut vod_col = Vec::with_capacity(10_000); + let mut series_col = Vec::with_capacity(50_000); + let mut vod_col = Vec::with_capacity(50_000); // preserve category_ids let (max_cat_id, existing_cat_ids) = load_old_category_ids(&path).await; @@ -276,26 +347,29 @@ pub async fn xtream_write_playlist( } let root_path = path.clone(); + let app_config = app_cfg.clone(); let write_errors = task::spawn_blocking(move || { let mut write_errors = vec![]; for (col_path, data) in [ - (get_collection_path(&root_path, storage_const::COL_CAT_LIVE), &cat_live_col), - (get_collection_path(&root_path, storage_const::COL_CAT_VOD), &cat_vod_col), - (get_collection_path(&root_path, storage_const::COL_CAT_SERIES), &cat_series_col), + (get_live_cat_collection_path(&root_path), &cat_live_col), + (get_vod_cat_collection_path(&root_path), &cat_vod_col), + (get_series_cat_collection_path(&root_path), &cat_series_col), ] { + let lock = app_config.file_locks.write_lock(&col_path); match json_write_documents_to_file(&col_path, data) { Ok(()) => {} Err(err) => { write_errors.push(format!("Persisting collection failed: {}: {err}", col_path.display())); } } + drop(lock); } write_errors }).await.map_err(|e| notify_err!(format!("Task panicked: {}", e)))?; errors.extend(write_errors); match write_playlists_to_file( - cfg, + app_cfg, &path, vec![ (XtreamCluster::Live, &live_col), @@ -304,7 +378,7 @@ pub async fn xtream_write_playlist( ], ).await { Ok(()) => { - if let Err(err) = xtream_garbage_collect(cfg, &target.name).await { + if let Err(err) = xtream_garbage_collect(app_cfg, &target.name).await { if err.kind() != ErrorKind::NotFound { errors.push(format!("Garbage collection failed:{err}")); } @@ -516,338 +590,60 @@ pub async fn xtream_write_series_info( app_config: &AppConfig, target_name: &str, series_info_id: u32, - content: &str, + content: &StreamProperties, ) -> Result<(), Error> { - let config = app_config.config.load(); - let target_path = try_option_ok!(get_target_storage_path(&config, target_name)); - let storage_path = try_option_ok!(xtream_get_storage_path(&config, target_name)); - let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths( - &storage_path, - XtreamCluster::Series - )); + if let StreamProperties::Series(_series) = content { + let config = app_config.config.load(); + let target_path = try_option_ok!(get_target_storage_path(&config, target_name)); + let storage_path = try_option_ok!(xtream_get_storage_path(&config, target_name)); + let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths( + &storage_path, + XtreamCluster::Series + )); - { - let _file_lock = app_config.file_locks.write_lock(&info_path).await; - let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?; - writer.write_doc(series_info_id, content).map_err(|_| str_to_io_error(&format!("failed to write xtream series info for target {target_name}")))?; - writer.store()?; - } - { - let target_id_mapping_file = get_target_id_mapping_file(&target_path); - let _file_lock = app_config.file_locks.write_lock(&target_id_mapping_file).await; - if let Ok(mut target_id_mapping) = BPlusTreeUpdate::::try_new(&target_id_mapping_file) { - if let Some(record) = target_id_mapping.query(&series_info_id) { - let new_record = record.copy_update_timestamp(); - let _ = target_id_mapping.update(&series_info_id, new_record); + { + let _file_lock = app_config.file_locks.write_lock(&info_path).await; + let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?; + writer.write_doc(series_info_id, content).map_err(|_| str_to_io_error(&format!("failed to write xtream series info for target {target_name}")))?; + writer.store()?; + } + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = app_config.file_locks.write_lock(&target_id_mapping_file).await; + if let Ok(mut target_id_mapping) = BPlusTreeUpdate::::try_new(&target_id_mapping_file) { + if let Some(record) = target_id_mapping.query(&series_info_id) { + let new_record = record.copy_update_timestamp(); + let _ = target_id_mapping.update(&series_info_id, new_record); + } } } - } - Ok(()) + Ok(()) + } else { + Err(std::io::Error::new(ErrorKind::InvalidData, "No series info data found!".to_string())) + } } pub async fn xtream_write_vod_info( app_config: &AppConfig, target_name: &str, virtual_id: u32, - content: &str, + content: &StreamProperties, ) -> Result<(), Error> { - let config = app_config.config.load(); - let storage_path = try_option_ok!(xtream_get_storage_path(&config, target_name)); - let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths(&storage_path, XtreamCluster::Video)); - { - let _file_lock = app_config.file_locks.write_lock(&info_path).await; - let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?; - writer.write_doc(virtual_id, content).map_err(|_| str_to_io_error(&format!("failed to write xtream vod info for target {target_name}")))?; - writer.store()?; - } - Ok(()) -} - -async fn xtream_get_series_info_mapping( - config: &AppConfig, - target_name: &str, - series_id: u32, -) -> Option { - xtream_get_info_mapping(config, target_name, series_id).await.filter(|id_record| !id_record.is_expired()) -} - -async fn xtream_get_info_mapping(app_config: &AppConfig, target_name: &str, info_id: u32) -> Option { - let config = app_config.config.load(); - let target_path = get_target_storage_path(&config, target_name)?; - - let target_id_mapping_file = get_target_id_mapping_file(&target_path); - let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; - BPlusTreeQuery::::try_new(&target_id_mapping_file).map_err(|err| { - error!("Could not load id mapping for target {target_name}: {err}"); - str_to_io_error(&format!("ID mapping load error for target {target_name}")) - }).ok().map(|mut tree| tree.query(&info_id))? -} - -// Reads the series info entry if exists -pub async fn xtream_load_series_info( - app_config: &AppConfig, - target_name: &str, - series_id: u32, -) -> Option { - xtream_get_series_info_mapping(app_config, target_name, series_id).await?; - let config = app_config.config.load(); - let storage_path = xtream_get_storage_path(&config, target_name)?; - - let (info_path, idx_path) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)?; - - if info_path.exists() && idx_path.exists() { + if let StreamProperties::Video(video) = content { + let config = app_config.config.load(); + let storage_path = try_option_ok!(xtream_get_storage_path(&config, target_name)); + let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths(&storage_path, XtreamCluster::Video)); { - let _file_lock = app_config.file_locks.read_lock(&info_path).await; - return match IndexedDocumentDirectAccess::read_indexed_item::(&info_path, &idx_path, &series_id) { - Ok(content) => Some(content), - Err(err) => { - error!("Failed to read series info for id {series_id} for {target_name}: {err}"); - None - } - }; - } - } - None -} -async fn xtream_get_vod_info_mapping( - config: &AppConfig, - target_name: &str, - vod_id: u32, -) -> Option { - xtream_get_info_mapping(config, target_name, vod_id).await - //.filter(|id_record| !id_record.is_expired()) -} - -// Reads the vod info entry if exists -pub async fn xtream_load_vod_info( - config: &AppConfig, - target_name: &str, - vod_id: u32, -) -> Option { - - // Check if the entry exists; if not, we don't need to look further. - xtream_get_vod_info_mapping(config, target_name, vod_id).await.as_ref()?; - // Entry exists, read db entry - let target_storage_path = xtream_get_storage_path(&config.config.load(), target_name)?; - - let (info_path, idx_path) = xtream_get_info_file_paths(&target_storage_path, XtreamCluster::Video)?; - - if info_path.exists() && idx_path.exists() { - { - let _file_lock = config.file_locks.read_lock(&info_path).await; - return IndexedDocumentDirectAccess::read_indexed_item::(&info_path, &idx_path, &vod_id).ok(); - } - } - None -} - -fn rewrite_xtream_vod_info

( - config: &AppConfig, - target: &ConfigTarget, - xtream_output: &XtreamTargetOutput, - pli: &P, - user: &ProxyUserCredentials, - doc: &mut Map, -) -> Result where - P: PlaylistEntry, -{ - // we need to update the info data. - if config.is_reverse_proxy_resource_rewrite_enabled() { - if let Some(Value::Object(info_data)) = doc.get_mut(crate::model::XC_TAG_INFO_DATA) { - normalize_release_date(info_data); // info data is muteable, so we do not need to clone it. - let item_type = pli.get_item_type(); - if user.proxy.is_reverse(item_type) && !target.is_force_redirect(item_type) { - let server_info = config.get_user_server_info(user); - let url = server_info.get_base_url(); - let resource_url = Some(format!("{url}/resource/movie/{}/{}/{}", user.username, user.password, pli.get_virtual_id())); - rewrite_doc_urls(resource_url.as_ref(), info_data, storage_const::INFO_REWRITE_FIELDS, crate::model::XC_INFO_RESOURCE_PREFIX); - // doc.insert(TAG_INFO_DATA, Value::Object(info_data)); - } - } - } - - // we need to update the movie data with virtual ids. - if let Some(Value::Object(movie_data)) = doc.get_mut(crate::model::XC_TAG_MOVIE_DATA) { - let stream_id = pli.get_virtual_id(); - let category_id = pli.get_category_id().unwrap_or(0); - movie_data.insert(crate::model::XC_TAG_STREAM_ID.to_string(), Value::Number(serde_json::value::Number::from(stream_id))); - movie_data.insert(crate::model::XC_TAG_CATEGORY_ID.to_string(), Value::Number(serde_json::value::Number::from(category_id))); - movie_data.insert(crate::model::XC_TAG_CATEGORY_IDS.to_string(), Value::Array(vec![Value::Number(serde_json::value::Number::from(category_id))])); - let options = xtream_mapping_option_from_target_options(target, xtream_output, config, user, None); - if options.skip_video_direct_source { - movie_data.insert(crate::model::XC_TAG_DIRECT_SOURCE.to_string(), Value::String(String::new())); - } else { - movie_data.insert( - crate::model::XC_TAG_DIRECT_SOURCE.to_string(), - Value::String(pli.get_provider_url().clone()), - ); - } - } - let result = serde_json::to_string(&doc).map_err(|_| str_to_io_error("Failed to serialize vod info"))?; - - Ok(result) -} - -pub fn rewrite_xtream_vod_info_content

( - config: &AppConfig, - target: &ConfigTarget, - xtream_output: &XtreamTargetOutput, - pli: &P, - user: &ProxyUserCredentials, - content: &str, -) -> Result where - P: PlaylistEntry, -{ - let mut doc = serde_json::from_str::>(content).map_err(|_| str_to_io_error("Failed to parse JSON content"))?; - rewrite_xtream_vod_info(config, target, xtream_output, pli, user, &mut doc) -} - -pub async fn write_and_get_xtream_vod_info

( - app_config: &AppConfig, - target: &ConfigTarget, - xtream_output: &XtreamTargetOutput, - pli: &P, - user: &ProxyUserCredentials, - content: &str, -) -> Result where - P: PlaylistEntry, -{ - let mut doc = serde_json::from_str::>(content).map_err(|_| str_to_io_error("Failed to parse JSON content"))?; - xtream_write_vod_info(app_config, target.name.as_str(), pli.get_virtual_id(), content).await.ok(); - rewrite_xtream_vod_info(app_config, target, xtream_output, pli, user, &mut doc) -} - -async fn rewrite_xtream_series_info

( - app_state: &Arc, - target: &ConfigTarget, - xtream_output: &XtreamTargetOutput, - pli: &P, - user: &ProxyUserCredentials, - doc: &mut Map, -) -> Result where - P: PlaylistEntry, -{ - let app_config = &app_state.app_config; - let config = app_config.config.load(); - let target_path = get_target_storage_path(&config, target.name.as_str()).ok_or_else(|| str_to_io_error(&format!("Could not find path for target {}", target.name)))?; - - let resource_url = if app_config.is_reverse_proxy_resource_rewrite_enabled() { - let item_type = pli.get_item_type(); - if user.proxy.is_reverse(item_type) && !target.is_force_redirect(item_type) { - let server_info = app_config.get_user_server_info(user); - let url = server_info.get_base_url(); - Some(format!("{url}/resource/series/{}/{}/{}", user.username, user.password, pli.get_virtual_id())) - } else { - None + let _file_lock = app_config.file_locks.write_lock(&info_path).await; + let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?; + writer.write_doc(virtual_id, video).map_err(|_| str_to_io_error(&format!("failed to write xtream vod info for target {target_name}")))?; + writer.store()?; } + Ok(()) } else { - None - }; - if resource_url.is_some() { - // we need to update the info data. - if let Some(Value::Object(info_data)) = doc.get_mut(crate::model::XC_TAG_INFO_DATA) { - rewrite_doc_urls(resource_url.as_ref(), info_data, storage_const::INFO_REWRITE_FIELDS, crate::model::XC_INFO_RESOURCE_PREFIX); - } - if let Some(Value::Array(seasons_data)) = doc.get_mut(crate::model::XC_TAG_SEASONS_DATA) { - for season_value in seasons_data { - if let Value::Object(season_doc) = season_value { - if let Some(season_provider_id) = season_doc.get(crate::model::XC_TAG_ID).and_then(get_u32_from_serde_value) { - let field_prefix = format!("{}{season_provider_id}_", crate::model::XC_SEASON_RESOURCE_PREFIX); - rewrite_doc_urls(resource_url.as_ref(), season_doc, storage_const::INFO_REWRITE_FIELDS, &field_prefix); - } - } - } - } + Err(std::io::Error::new(ErrorKind::InvalidData, "No video info data found!".to_string())) } - - let episodes = doc.get_mut(crate::model::XC_TAG_EPISODES).and_then(Value::as_object_mut).ok_or_else(|| str_to_io_error("No episodes found in content"))?; - - let virtual_id = pli.get_virtual_id(); - { - let (mut target_id_mapping, file_lock) = get_target_id_mapping(app_config, &target_path).await; - let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, None); - - let use_memory_cache = target.use_memory_cache; - let mut id_mapping_records = vec![]; - - let provider_url = pli.get_provider_url(); - for episode_list in episodes.values_mut().filter_map(Value::as_array_mut) { - for episode in episode_list.iter_mut().filter_map(Value::as_object_mut) { - if let Some(episode_provider_id) = episode.get(crate::model::XC_TAG_ID).and_then(get_u32_from_serde_value) - { - let uuid = generate_playlist_uuid(&hex_encode(&pli.get_uuid()), &episode_provider_id.to_string(), PlaylistItemType::Series, &provider_url); - let episode_virtual_id = target_id_mapping.get_and_update_virtual_id( - &uuid, - episode_provider_id, - PlaylistItemType::Series, - virtual_id, - ); - - if use_memory_cache { - let record = VirtualIdRecord::new(episode_provider_id, episode_virtual_id, PlaylistItemType::Series, virtual_id, uuid); - id_mapping_records.push(record); - } - - episode.insert(crate::model::XC_TAG_ID.to_string(), Value::String(episode_virtual_id.to_string())); - if resource_url.is_some() { - // we need to update the info data. - if let Some(Value::Object(info_data)) = episode.get_mut(crate::model::XC_TAG_INFO_DATA) { - let field_prefix = format!("{}{episode_provider_id}_", crate::model::XC_INFO_RESOURCE_PREFIX_EPISODE); - rewrite_doc_urls(resource_url.as_ref(), info_data, storage_const::INFO_REWRITE_FIELDS, &field_prefix); - } - } - } - if options.skip_series_direct_source { - episode.insert(crate::model::XC_TAG_DIRECT_SOURCE.to_string(), Value::String(String::new())); - } - } - } - - let result = target_id_mapping.persist(); - drop(file_lock); - drop(target_id_mapping); - - if let Err(err) = result { - error!("{err}"); - } else if use_memory_cache && !id_mapping_records.is_empty() { - app_state.playlists.update_target_id_mapping(target, id_mapping_records).await; - } - } - let result = serde_json::to_string(&doc).map_err(|_| str_to_io_error("Failed to serialize updated series info"))?; - - Ok(result) -} - -pub async fn rewrite_xtream_series_info_content

( - app_state: &Arc, - target: &ConfigTarget, - xtream_output: &XtreamTargetOutput, - pli_series_info: &P, - user: &ProxyUserCredentials, - content: &str, -) -> Result where - P: PlaylistEntry, -{ - let mut doc = serde_json::from_str::>(content).map_err(|_| str_to_io_error("Failed to parse JSON content"))?; - rewrite_xtream_series_info(app_state, target, xtream_output, pli_series_info, user, &mut doc).await -} - -pub async fn write_and_get_xtream_series_info

( - app_state: &Arc, - target: &ConfigTarget, - xtream_output: &XtreamTargetOutput, - pli_series_info: &P, - user: &ProxyUserCredentials, - content: &str, -) -> Result where - P: PlaylistEntry, -{ - let mut doc = serde_json::from_str::>(content).map_err(|_| str_to_io_error("Failed to parse JSON content"))?; - let virtual_id = pli_series_info.get_virtual_id(); - let app_config = &app_state.app_config; - xtream_write_series_info(app_config, target.name.as_str(), virtual_id, content).await.ok(); - rewrite_xtream_series_info(app_state, target, xtream_output, pli_series_info, user, &mut doc).await } pub async fn xtream_get_input_info( @@ -855,13 +651,15 @@ pub async fn xtream_get_input_info( input: &ConfigInput, provider_id: u32, cluster: XtreamCluster, -) -> Option { +) -> Option { if let Ok(Some((info_path, idx_path))) = get_input_storage_path(&input.name, &cfg.config.load().working_dir).map(|storage_path| xtream_get_info_file_paths(&storage_path, cluster)) { let _file_lock = cfg.file_locks.read_lock(&info_path).await; - if let Ok(content) = IndexedDocumentDirectAccess::read_indexed_item::(&info_path, &idx_path, &provider_id) { - return Some(content); - } + return match cluster { + XtreamCluster::Live => IndexedDocumentDirectAccess::read_indexed_item::(&info_path, &idx_path, &provider_id).ok().map(StreamProperties::Live), + XtreamCluster::Video => IndexedDocumentDirectAccess::read_indexed_item::(&info_path, &idx_path, &provider_id).ok().map(Box::new).map(StreamProperties::Video), + XtreamCluster::Series => IndexedDocumentDirectAccess::read_indexed_item::(&info_path, &idx_path, &provider_id).ok().map(Box::new).map(StreamProperties::Series), + }; } None } @@ -911,75 +709,6 @@ pub async fn xtream_update_input_info_file( } } -pub async fn xtream_update_input_vod_record_from_wal_file( - cfg: &AppConfig, - input: &ConfigInput, - wal_path: &Path, -) -> Result<(), TuliproxError> { - let config = cfg.config.load(); - let record_path = get_input_storage_path(&input.name, &config.working_dir).map(|storage_path| xtream_get_record_file_path(&storage_path, PlaylistItemType::Video)) - .map_err(|err| notify_err!(format!("Error accessing storage path: {err}"))) - .and_then(|opt| opt.ok_or_else(|| notify_err!(format!("Error accessing storage path for input: {}", &input.name))))?; - - { - let _file_lock = cfg.file_locks.write_lock(&record_path).await; - let mut reader = async_file_reader(async_open_readonly_file(wal_path).await.map_err(|err| notify_err!(format!("Could not read vod wal info {err}")))?); - let mut provider_id_bytes = [0u8; 4]; - let mut tmdb_id_bytes = [0u8; 4]; - let mut ts_bytes = [0u8; 8]; - let mut tree_record_index: BPlusTree = BPlusTree::load(&record_path).unwrap_or_else(|_| BPlusTree::new()); - - loop { - if reader.read_exact(&mut provider_id_bytes).await.is_err() { - break; // End of file - } - let provider_id = u32::from_le_bytes(provider_id_bytes); - - if reader.read_exact(&mut tmdb_id_bytes).await.is_err() { - error!("Unexpected EOF after reading provider_id {provider_id} for VOD record."); - break; - } - let tmdb_id = u32::from_le_bytes(tmdb_id_bytes); - - if reader.read_exact(&mut ts_bytes).await.is_err() { - error!("Unexpected EOF after reading tmdb_id for VOD record with provider_id {provider_id}."); - break; - } - let ts = u64::from_le_bytes(ts_bytes); - - // Read the date string length as a 4-byte u32. - let mut len_bytes = [0u8; 4]; - if reader.read_exact(&mut len_bytes).await.is_err() { - error!("Unexpected EOF when reading release_date length for VOD record with provider_id {provider_id}."); - break; - } - let len = u32::from_le_bytes(len_bytes) as usize; - - let release_date = if len > 0 { - let mut date_buffer = vec![0u8; len]; - if reader.read_exact(&mut date_buffer).await.is_err() { - error!("Unexpected EOF when reading release_date string for VOD record with provider_id {provider_id}."); - break; - } - String::from_utf8(date_buffer).ok() - } else { - // If length is 0, set release_date to None - None - }; - - tree_record_index.insert(provider_id, InputVodInfoRecord { tmdb_id, ts, release_date }); - } - - tree_record_index.store(&record_path).map_err(|err| notify_err!(format!("Could not store vod record info {err}")))?; - - drop(reader); - if let Err(err) = tokio::fs::remove_file(wal_path).await { - error!("Failed to delete record WAL file for vod {err}"); - } - Ok(()) - } -} - pub async fn xtream_update_input_series_record_from_wal_file( cfg: &AppConfig, input: &ConfigInput, @@ -1015,58 +744,6 @@ pub async fn xtream_update_input_series_record_from_wal_file( } } -pub async fn xtream_update_input_series_episodes_record_from_wal_file( - cfg: &AppConfig, - input: &ConfigInput, - wal_path: &Path, -) -> Result<(), TuliproxError> { - let config = cfg.config.load(); - let record_path = get_input_storage_path(&input.name, &config.working_dir).map(|storage_path| xtream_get_record_file_path(&storage_path, PlaylistItemType::Series)) - .map_err(|err| notify_err!(format!("Error accessing storage path: {err}"))) - .and_then(|opt| opt.ok_or_else(|| notify_err!(format!("Error accessing storage path for input: {}", &input.name))))?; - { - let _file_lock = cfg.file_locks.write_lock(&record_path).await; - let mut reader = async_file_reader(async_open_readonly_file(wal_path).await.map_err(|err| notify_err!(format!("Could not read series episode wal info {err}")))?); - let mut provider_id_bytes = [0u8; 4]; - let mut len_bytes = [0u8; 4]; - let mut tree_record_index: BPlusTree = BPlusTree::load(&record_path).unwrap_or_else(|_| BPlusTree::new()); - let mut buffer = vec![0u8; 4096]; - loop { - if reader.read_exact(&mut provider_id_bytes).await.is_err() { - break; // End of file - } - let provider_id = u32::from_le_bytes(provider_id_bytes); - if reader.read_exact(&mut len_bytes).await.is_err() { - break; // End of file - } - let len = usize::try_from(u32::from_le_bytes(len_bytes)).unwrap_or(0); - if len == 0 { - break; - } - if len > buffer.len() { - buffer = vec![0u8; len]; - } - if reader.read_exact(&mut buffer[0..len]).await.is_err() { - break; - } - match bincode_deserialize(&buffer[0..len]) { - Ok(episode) => { - tree_record_index.insert(provider_id, episode); - } - Err(err) => { - error!("Failed to delete deserialize record WAL file for series episode {err}"); - } - } - } - tree_record_index.store(&record_path).map_err(|err| notify_err!(format!("Could not store series episode record info {err}")))?; - drop(reader); - if let Err(err) = tokio::fs::remove_file(wal_path).await { - error!("Failed to delete record WAL file for series episode {err}"); - } - Ok(()) - } -} - pub async fn iter_raw_xtream_playlist(app_config: &AppConfig, target: &ConfigTarget, cluster: XtreamCluster) -> Option<(FileReadGuard, impl Iterator)> { let config = app_config.config.load(); if let Some(storage_path) = xtream_get_storage_path(&config, target.name.as_str()) { @@ -1134,17 +811,6 @@ pub async fn write_series_info_to_wal_file(provider_id: u32, ts: u64, content: & Ok(()) } -pub async fn write_vod_info_to_wal_file(provider_id: u32, content: &str, info_record: &InputVodInfoRecord, - content_write: &mut tokio::io::BufWriter<&mut tokio::fs::File>, - record_writer: &mut tokio::io::BufWriter<&mut tokio::fs::File>) -> std::io::Result<()> { - let encoded_content = encode_info_content_for_wal_file(provider_id, content)?; - let encoded_record = encode_vod_info_record_for_wal_file(provider_id, info_record)?; - content_write.write_all(&encoded_content).await?; - record_writer.write_all(&encoded_record).await?; - Ok(()) -} - - fn encode_info_content_for_wal_file(provider_id: u32, content: &str) -> std::io::Result> { let length = u32::try_from(content.len()).map_err(to_io_error)?; let mut buffer = Vec::with_capacity(8 + content.len()); @@ -1156,35 +822,207 @@ fn encode_info_content_for_wal_file(provider_id: u32, content: &str) -> std::io: Ok(buffer) } - -fn encode_vod_info_record_for_wal_file(provider_id: u32, record: &InputVodInfoRecord) -> std::io::Result> { - let estimated_capacity = 30; - let mut buffer = Vec::with_capacity(estimated_capacity); - - buffer.extend_from_slice(&provider_id.to_le_bytes()); - buffer.extend_from_slice(&record.tmdb_id.to_le_bytes()); - buffer.extend_from_slice(&record.ts.to_le_bytes()); - - match &record.release_date { - Some(date_str) => { - // Write the length as a 4-byte u32 to prevent overflow. - let len = u32::try_from(date_str.len()).map_err(to_io_error)?; - buffer.extend_from_slice(&len.to_le_bytes()); - buffer.extend_from_slice(date_str.as_bytes()); - } - None => { - // Write a length of 0 as 4 bytes. - buffer.extend_from_slice(&0u32.to_le_bytes()); - } - } - - Ok(buffer) -} - - fn encode_series_info_record_for_wal_file(provider_id: u32, ts: u64) -> Vec { let mut buffer = Vec::with_capacity(12); buffer.extend_from_slice(&provider_id.to_le_bytes()); buffer.extend_from_slice(&ts.to_le_bytes()); buffer } + +#[allow(clippy::too_many_lines)] +pub async fn persist_input_xtream_playlist(app_config: &Arc, storage_path: &Path, + playlist: Vec) -> (Vec, Option) { + let mut errors = Vec::new(); + + let mut fetched_categories = PlaylistScratch::>::new(1_000); + let mut fetched_scratch = PlaylistScratch::>::new(50_000); + let mut stored_scratch = PlaylistScratch::>::new(50_000); + + // load + for cluster in XTREAM_CLUSTER { + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); + if let Ok(true) = tokio::fs::try_exists(&xtream_path).await { + let file_lock = app_config.file_locks.read_lock(&xtream_path).await; + let stored_entries = stored_scratch.get_mut(cluster); + if let Ok(reader) = IndexedDocumentIterator::::new(&xtream_path, &idx_path) { + for (doc, _) in reader { + if let Ok(provider_id) = doc.header.id.parse::() { + stored_entries.insert(provider_id, doc); + } + } + } + drop(file_lock); + } + } + + let mut groups = IndexMap::new(); + + for mut plg in playlist { + if !&plg.channels.is_empty() { + fetched_categories.get_mut(plg.xtream_cluster).push(json!(CategoryEntry { + category_id: plg.id, + category_name: plg.title.clone(), + parent_id: 0 + })); + + let channels = std::mem::take(&mut plg.channels); + for mut pli in channels { + let stored_col = stored_scratch.get_mut(plg.xtream_cluster); + let fetched_col = fetched_scratch.get_mut(plg.xtream_cluster); + + if let Ok(provider_id) = pli.header.id.parse::() { + if let Some(stored_pli) = stored_col.get_mut(&provider_id) { + if let (Some(new_stream_props), Some(old_stream_props)) = (&mut pli.header.additional_properties, stored_pli.header.additional_properties.take()) { + if !needs_update_info_details(new_stream_props, &old_stream_props) { + match (new_stream_props, old_stream_props) { + (StreamProperties::Video(value_1), StreamProperties::Video(value_2)) => { + value_1.details = value_2.details; + } + (StreamProperties::Series(value_1), StreamProperties::Series(value_2)) => { + value_1.details = value_2.details; + } + _ => {} + } + } + } + } + } + fetched_col.push(pli); + } + groups.insert(plg.id, plg); + } + } + + let mut processed_scratch = PlaylistScratch::>::new(0); + for xc in XTREAM_CLUSTER { + processed_scratch.set(xc, if !stored_scratch.is_empty(xc) && fetched_scratch.is_empty(xc) { + stored_scratch.take(xc).into_values().collect() + } else { + fetched_scratch.take(xc) + }); + } + drop(stored_scratch); + drop(fetched_scratch); + + let root_path = storage_path.to_path_buf(); + let app_cfg = app_config.clone(); + let write_errors = task::spawn_blocking(move || { + let mut write_errors = vec![]; + for cluster in XTREAM_CLUSTER { + let col_path = match cluster { + XtreamCluster::Live => get_collection_path(&root_path, storage_const::COL_CAT_LIVE), + XtreamCluster::Video => get_collection_path(&root_path, storage_const::COL_CAT_VOD), + XtreamCluster::Series => get_collection_path(&root_path, storage_const::COL_CAT_SERIES), + }; + let data = fetched_categories.get_mut(cluster); + let lock = app_cfg.file_locks.write_lock(&col_path); + if let Err(err) = json_write_documents_to_file(&col_path, data) { + write_errors.push(format!( + "Persisting collection failed: {}: {err}", + col_path.display() + )); + } + drop(lock); + } + write_errors + }).await.map_err(|e| notify_err!(format!("Task panicked: {}", e))); + + for cluster in XTREAM_CLUSTER { + let data = processed_scratch.get(cluster); + match write_playlists_to_file_2( + app_config, + storage_path, + cluster, + data, + ).await { + Ok(()) => { + + // TODO GARBAGE collect + + // if let Err(err) = xtream_garbage_collect(app_config, &target.name).await { + // if err.kind() != ErrorKind::NotFound { + // errors.push(format!("Garbage collection failed:{err}")); + // } + // } + } + Err(err) => { + errors.push(format!("Persisting collection failed:{err}")); + } + } + } + + match write_errors { + Ok(write_err) => errors.extend(write_err), + Err(err) => errors.push(err.to_string()), + } + + for xc in XTREAM_CLUSTER { + let col = processed_scratch.take(xc); + for item in col { + groups + .entry(item.header.category_id) + .or_insert_with(|| PlaylistGroup { + id: item.header.category_id, + title: item.header.group.clone(), + channels: Vec::new(), + xtream_cluster: item.header.xtream_cluster, + }) + .channels + .push(item); + } + } + + let result = groups.into_iter().map(|(_, group)| group).collect(); + + let err = if errors.is_empty() { + None + } else { + Some(create_tuliprox_error!(TuliproxErrorKind::Notify, "{}", errors.join("\n"))) + }; + + (result, err) +} + +// Checks if the info has changed after the last update +fn needs_update_info_details( + new_stream_props: &StreamProperties, + old_stream_props: &StreamProperties, +) -> bool { + let new_modified = new_stream_props.get_last_modified(); + let old_modified = old_stream_props.get_last_modified(); + + match (new_modified, old_modified) { + (Some(new_ts), Some(old_ts)) => new_ts > old_ts, + (None, Some(_)) => true, + _ => false, + } +} + +async fn persist_input_info(app_config: &Arc, storage_path: &Path, cluster: XtreamCluster, + input_name: &str, provider_id: u32, props: T) -> Result<(), Error> +where + T: serde::Serialize, +{ + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); + if let Ok(true) = tokio::fs::try_exists(&xtream_path).await { + { + let _file_lock = app_config.file_locks.write_lock(&xtream_path).await; + let mut writer = IndexedDocumentWriter::new_append(xtream_path.clone(), idx_path)?; + writer.write_doc(provider_id, &props).map_err(|_| str_to_io_error(&format!("failed to write vod info for input {input_name}")))?; + writer.store()?; + } + } + Ok(()) +} + +pub async fn persists_input_vod_info(app_config: &Arc, storage_path: &Path, + cluster: XtreamCluster, input_name: &str, provider_id: u32, + props: &VideoStreamProperties) -> Result<(), Error> { + persist_input_info::<&VideoStreamProperties>(app_config, storage_path, cluster, input_name, provider_id, props).await +} + +pub async fn persists_input_series_info(app_config: &Arc, storage_path: &Path, + cluster: XtreamCluster, input_name: &str, provider_id: u32, + props: &SeriesStreamProperties) -> Result<(), Error> { + persist_input_info::<&SeriesStreamProperties>(app_config, storage_path, cluster, input_name, provider_id, props).await +} \ No newline at end of file diff --git a/backend/src/tools/lru_cache.rs b/backend/src/tools/lru_cache.rs index d666ab1c4..14f2dfaca 100644 --- a/backend/src/tools/lru_cache.rs +++ b/backend/src/tools/lru_cache.rs @@ -1,11 +1,16 @@ -use crate::utils::{traverse_dir}; -use shared::utils::{hash_string_as_hex, human_readable_byte_size, sanitize_sensitive_info}; +use crate::utils::{decode_base64_string, encode_base64_hash, encode_base64_string, traverse_dir}; +use shared::utils::{human_readable_byte_size, sanitize_sensitive_info}; use log::{debug, error, info, trace}; use std::collections::{HashMap, VecDeque}; use std::fs; use std::path::PathBuf; use crate::utils::{trace_if_enabled}; +#[inline] +fn encode_cache_key(key: &str) -> String { + encode_base64_hash(key) +} + /// `LRUResourceCache` /// /// A least-recently-used (LRU) file-based resource cache that stores files in a directory on disk, @@ -22,7 +27,7 @@ pub struct LRUResourceCache { capacity: usize, // Maximum size in bytes cache_dir: PathBuf, current_size: usize, // Current size in bytes - cache: HashMap, + cache: HashMap, usize)>, usage_order: VecDeque, } @@ -37,7 +42,7 @@ impl LRUResourceCache { capacity, cache_dir: PathBuf::from(cache_dir), current_size: 0, - cache: HashMap::::new(), + cache: HashMap::, usize)>::with_capacity(4096), usage_order: VecDeque::new(), } } @@ -53,15 +58,22 @@ impl LRUResourceCache { pub fn scan(&mut self) -> std::io::Result<()> { let mut visit = |entry: &std::fs::DirEntry, metadata: &std::fs::Metadata| { let path = entry.path(); - if let Some(file_name) = path.file_name() { - let key = String::from(file_name.to_string_lossy()); + if let Some(os_file_name) = path.file_name() { + let file_name = String::from(os_file_name.to_string_lossy()); + let (key, mime_type) = if let Some((part1, part2)) = file_name.split_once('.') { + (part1.to_string(), String::from_utf8(decode_base64_string(part2)).ok()) + } else { + (file_name.clone(), None) + }; + let file_size = usize::try_from(metadata.len()).unwrap_or(0); // we need to duplicate because of closure we cant call insert_to_cache { // insert_to_cache + let mut path = self.cache_dir.clone(); - path.push(&key); + path.push(&file_name); trace!("Added file to cache: {}", &path.to_string_lossy()); - self.cache.insert(key.clone(), (path.clone(), file_size)); + self.cache.insert(key.clone(), (path.clone(), mime_type, file_size)); self.usage_order.push_back(key); self.current_size += file_size; } @@ -83,48 +95,61 @@ impl LRUResourceCache { /// - `file_size`: The size of the file in bytes. /// - Returns: /// - The `PathBuf` where the file is stored. - pub fn add_content(&mut self, url: &str, file_size: usize) -> std::io::Result { - let key = hash_string_as_hex(url); - let path = self.insert_to_cache(key, file_size); + pub fn add_content(&mut self, url: &str, mime_type: Option, file_size: usize) -> std::io::Result { + let key = encode_cache_key(url); + // let (key, mime_type) = if let Some((part1, part2)) = file_name.split_once(".") { + // (part1.to_string(), String::from_utf8(decode_base64_string(part2)).ok()) + // } else { + // (key, None) + // }; + let path = self.insert_to_cache(key, mime_type, file_size); if self.current_size > self.capacity { self.evict_if_needed(); } Ok(path) } - fn insert_to_cache(&mut self, key: String, file_size: usize) -> PathBuf { - let mut path = self.cache_dir.clone(); - path.push(&key); + fn insert_to_cache(&mut self, key: String, mime_type: Option, file_size: usize) -> PathBuf { + let path = self.get_store_path(&key, mime_type.as_deref()); debug!("Added file to cache: {}", &path.to_string_lossy()); - self.cache.insert(key.clone(), (path.clone(), file_size)); + self.cache.insert(key.clone(), (path.clone(), mime_type, file_size)); self.usage_order.push_back(key); self.current_size += file_size; path } - pub fn store_path(&self, url: &str) -> PathBuf { - let key = hash_string_as_hex(url); + pub fn store_path(&self, url: &str, mime_type: Option<&str>) -> PathBuf { + self.get_store_path(&encode_cache_key(url), mime_type) + } + + fn get_store_path(&self, cache_key: &str, mime_type: Option<&str>) -> PathBuf { + let key = if let Some(mime) = mime_type { + format!("{cache_key}.{}", encode_base64_string(mime.as_bytes())) + } else { + cache_key.to_string() + }; let mut path = self.cache_dir.clone(); path.push(&key); path } + /// - Retrieves a file from the cache if it exists. /// - Moves the file's key to the end of the usage queue to mark it as recently used. /// - Arguments: /// - `url`: The unique identifier for the file. /// - Returns: /// - The `PathBuf` of the file if it exists; `None` otherwise. - pub fn get_content(&mut self, url: &str) -> Option { - let key = hash_string_as_hex(url); + pub fn get_content(&mut self, url: &str) -> Option<(PathBuf, Option)> { + let key = encode_cache_key(url); { - if let Some((path, size)) = self.cache.get(&key) { + if let Some((path, mime_type, size)) = self.cache.get(&key) { if path.exists() { trace_if_enabled!("Responding resource from cache with key: {key} for url: {}", sanitize_sensitive_info(url)); // Move to the end of the queue self.usage_order.retain(|k| k != &key); // remove from queue self.usage_order.push_back(key); // add to the to end - return Some(path.clone()); + return Some((path.clone(), mime_type.clone())); } { trace_if_enabled!("Cache inconsistency: file missing for key: {key}, url: {}", sanitize_sensitive_info(url)); @@ -142,7 +167,7 @@ impl LRUResourceCache { // if the cache size is to small and one element exceeds the size than the cache won't work, we ignore this while self.current_size > self.capacity { if let Some(oldest_file) = self.usage_order.pop_front() { - if let Some((file, size)) = self.cache.remove(&oldest_file) { + if let Some((file, _mime_type, size)) = self.cache.remove(&oldest_file) { self.current_size -= size; if let Err(err) = fs::remove_file(&file) { error!("Failed to delete cached file {} {err}", file.to_string_lossy()); diff --git a/backend/src/utils/binary_utils.rs b/backend/src/utils/binary_utils.rs new file mode 100644 index 000000000..33296e233 --- /dev/null +++ b/backend/src/utils/binary_utils.rs @@ -0,0 +1,22 @@ +use std::io; +use log::error; +use shared::error::to_io_error; + +#[inline] +pub fn binary_serialize(value: &T) -> io::Result> +where + T: ?Sized + serde::Serialize, +{ + rmp_serde::to_vec(value).map_err(to_io_error) +} + +#[inline] +pub fn binary_deserialize(value: &[u8]) -> io::Result +where + T: for<'a> serde::Deserialize<'a>, +{ + rmp_serde::from_slice(value).map_err(|e| { + error!("Failed to decode {e}"); + to_io_error(e) + }) +} diff --git a/backend/src/utils/bincode_utils.rs b/backend/src/utils/bincode_utils.rs deleted file mode 100644 index 76ed80ee0..000000000 --- a/backend/src/utils/bincode_utils.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::io; -use log::error; -use shared::error::to_io_error; - -#[inline] -pub fn bincode_serialize(value: &T) -> io::Result> -where - T: ?Sized + serde::Serialize, -{ - bincode::serde::encode_to_vec(value, bincode::config::legacy()).map_err(to_io_error) -} - -#[inline] -pub fn bincode_deserialize(value: &[u8]) -> io::Result -where - T: for<'a> serde::Deserialize<'a>, -{ - match bincode::serde::decode_from_slice(value, bincode::config::legacy()) { - Ok((instance, _size)) => Ok(instance), - Err(e) => { - error!("Failed to decode {e}"); - Err(to_io_error(e)) - }, - } -} diff --git a/backend/src/utils/crypto_utils.rs b/backend/src/utils/crypto_utils.rs index 74af65e99..b0b8f4ec2 100644 --- a/backend/src/utils/crypto_utils.rs +++ b/backend/src/utils/crypto_utils.rs @@ -3,11 +3,16 @@ use openssl::symm::{Cipher, Crypter, Mode}; use rand::{RngCore, rngs::OsRng, TryRngCore}; use shared::error::{TuliproxError, TuliproxErrorKind}; -fn encode_base64_string(input: &[u8]) -> String { +pub fn encode_base64_hash(text: &str) -> String { + let hash = blake3::hash(text.as_bytes()); + encode_base64_string(hash.as_bytes()) +} + +pub fn encode_base64_string(input: &[u8]) -> String { general_purpose::URL_SAFE_NO_PAD.encode(input) } -fn decode_base64_string(input: &str) -> Vec { +pub fn decode_base64_string(input: &str) -> Vec { general_purpose::URL_SAFE_NO_PAD.decode(input).unwrap_or_else(|_| input.as_bytes().to_vec()) } diff --git a/backend/src/utils/mod.rs b/backend/src/utils/mod.rs index 9bf79a555..4494602c6 100644 --- a/backend/src/utils/mod.rs +++ b/backend/src/utils/mod.rs @@ -7,11 +7,11 @@ mod step_measure; mod logging; mod trakt; mod json_utils; -mod bincode_utils; +mod binary_utils; mod telegram; mod geoip; -pub use self::bincode_utils::*; +pub use self::binary_utils::*; pub use self::logging::*; pub use self::trakt::*; pub use self::telegram::*; diff --git a/backend/src/utils/network/m3u.rs b/backend/src/utils/network/m3u.rs index 058c69185..5a468d6b7 100644 --- a/backend/src/utils/network/m3u.rs +++ b/backend/src/utils/network/m3u.rs @@ -6,7 +6,7 @@ use crate::processing::parser::m3u; use crate::utils::prepare_file_path; use crate::utils::request; -pub async fn get_m3u_playlist(client: &reqwest::Client, cfg: &Arc, input: &Arc) -> (Vec, Vec) { +pub async fn download_m3u_playlist(client: &reqwest::Client, cfg: &Arc, input: &Arc) -> (Vec, Vec) { let working_dir = &cfg.working_dir; let input_source: InputSource = { match input.staged.as_ref() { diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 1dcdce9cb..0ee1e9fa6 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -108,7 +108,6 @@ pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &Con } } - pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option) -> Result { debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url)); diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index 5757d239b..2649bb31f 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -1,16 +1,20 @@ +use std::collections::HashMap; use crate::api::model::AppState; use crate::messaging::send_message; -use crate::model::{is_input_expired, Config, ConfigInput, ConfigTarget, XtreamLoginInfo}; +use crate::model::{is_input_expired, xtream_mapping_option_from_target_options, Config, ConfigInput, ConfigTarget, XtreamLoginInfo}; use crate::model::{InputSource, ProxyUserCredentials}; use crate::processing::parser::xtream; -use crate::repository::xtream_repository; -use crate::repository::xtream_repository::{rewrite_xtream_series_info_content, rewrite_xtream_vod_info_content, xtream_get_input_info}; +use crate::processing::parser::xtream::parse_xtream_series_info; +use crate::repository::playlist_repository::{get_target_id_mapping, rewrite_provider_series_info_episode_virtual_id, ProviderEpisodeKey}; +use crate::repository::storage::{get_input_storage_path, get_target_storage_path}; +use crate::repository::target_id_mapping::VirtualIdRecord; +use crate::repository::xtream_repository::{persists_input_series_info, persists_input_vod_info, write_playlist_item_to_file}; use crate::utils::request; use chrono::{DateTime, Utc}; use log::{error, info, warn}; -use shared::error::{str_to_io_error, TuliproxError}; -use shared::model::{MsgKind, PlaylistEntry, PlaylistGroup, ProxyUserStatus, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{extract_extension_from_url, get_i64_from_serde_value, get_string_from_serde_value}; +use shared::error::{str_to_io_error, to_io_error, TuliproxError}; +use shared::model::{MsgKind, PlaylistEntry, PlaylistGroup, ProxyUserStatus, SeriesStreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamSeriesInfo, XtreamVideoInfo}; +use shared::utils::{extract_extension_from_url, get_i64_from_serde_value, get_string_from_serde_value, sanitize_sensitive_info}; use std::io::Error; use std::str::FromStr; use std::sync::Arc; @@ -54,57 +58,117 @@ pub async fn get_xtream_stream_info_content(client: &reqwest::Client, input: &In } #[allow(clippy::too_many_arguments)] -pub async fn get_xtream_stream_info

(client: &reqwest::Client, - app_state: &Arc, - user: &ProxyUserCredentials, - input: &ConfigInput, - target: &ConfigTarget, - pli: &P, - info_url: &str, - cluster: XtreamCluster) -> Result -where - P: PlaylistEntry, -{ +pub async fn get_xtream_stream_info(client: &reqwest::Client, + app_state: &Arc, + user: &ProxyUserCredentials, + input: &ConfigInput, + target: &ConfigTarget, + pli: &XtreamPlaylistItem, + info_url: &str, + cluster: XtreamCluster) -> Result { let xtream_output = target.get_xtream_output().ok_or_else(|| Error::other("Unexpected error, missing xtream output"))?; let app_config = &app_state.app_config; + let server_info = app_config.get_user_server_info(user); + let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str())); - if cluster == XtreamCluster::Series { - if let Some(content) = xtream_repository::xtream_load_series_info(app_config, target.name.as_str(), pli.get_virtual_id()).await { - // Deliver existing target content - return rewrite_xtream_series_info_content(app_state, target, xtream_output, pli, user, &content).await; - } - - // Check if the content has been resolved - if xtream_output.resolve_series { - if let Some(provider_id) = pli.get_provider_id() { - if let Some(content) = xtream_get_input_info(app_config, input, provider_id, XtreamCluster::Series).await { - return xtream_repository::write_and_get_xtream_series_info(app_state, target, xtream_output, pli, user, &content).await; - } - } - } - } else if cluster == XtreamCluster::Video { - if let Some(content) = xtream_repository::xtream_load_vod_info(app_config, target.name.as_str(), pli.get_virtual_id()).await { - // Deliver existing target content - return rewrite_xtream_vod_info_content(app_config, target, xtream_output, pli, user, &content); - } - // Check if the content has been resolved - if xtream_output.resolve_vod { - if let Some(provider_id) = pli.get_provider_id() { - if let Some(content) = xtream_get_input_info(app_config, input, provider_id, XtreamCluster::Video).await { - return xtream_repository::write_and_get_xtream_vod_info(app_config, target, xtream_output, pli, user, &content).await; - } - } - } + if let Some(content) = pli.get_resolved_info_document(&options) { + return serde_json::to_string(&content).map_err(to_io_error); } let input_source = InputSource::from(input).with_url(info_url.to_owned()); if let Ok(content) = get_xtream_stream_info_content(client, &input_source).await { - return match cluster { - XtreamCluster::Live => Ok(content), - XtreamCluster::Video => xtream_repository::write_and_get_xtream_vod_info(app_config, target, xtream_output, pli, user, &content).await, - XtreamCluster::Series => xtream_repository::write_and_get_xtream_series_info(app_state, target, xtream_output, pli, user, &content).await, - }; + if let Some(provider_id) = pli.get_provider_id() { + match cluster { + XtreamCluster::Live => {} + XtreamCluster::Video => { + let working_dir = &app_config.config.load().working_dir; + if let Ok(storage_path) = get_input_storage_path(&input.name, working_dir) { + if let Ok(info) = serde_json::from_str::(&content) { + let video_stream_props = VideoStreamProperties::from_info(&info, pli); + if let Err(err) = persists_input_vod_info(&app_state.app_config, &storage_path, cluster, &input.name, provider_id, &video_stream_props).await { + error!("Failed to persist video stream for input {}: {err}", &input.name); + } + + if let Err(err) = write_playlist_item_to_file(app_config, &target.name, pli).await { + error!("Failed to persist video stream: {err}"); + } + + if target.use_memory_cache { + app_state.playlists.update_playlist_items(target, vec![pli]).await; + } + } + } + } + XtreamCluster::Series => { + let working_dir = &app_config.config.load().working_dir; + let group = pli.get_group(); + let series_name = pli.get_name(); + + if let Ok(storage_path) = get_input_storage_path(&input.name, working_dir) { + if let Ok(info) = serde_json::from_str::(&content) { + let series_stream_props = SeriesStreamProperties::from_info(&info, pli); + let _ = persists_input_series_info(app_config, &storage_path, cluster, &input.name, provider_id, &series_stream_props).await; + if let Some(mut episodes) = parse_xtream_series_info(&pli.get_uuid(), &series_stream_props, &group, &series_name, input) { + let config = &app_state.app_config.config.load(); + match get_target_storage_path(config, target.name.as_str()) { + None => { + error!("Failed to get target storage path {}. Cant save episodes", &target.name); + } + Some(target_path) => { + let mut in_memory_updates = Vec::new(); + let mut provider_series: HashMap> = HashMap::new(); + { + let (mut target_id_mapping, _file_lock) = get_target_id_mapping(&app_state.app_config, &target_path).await; + if let Some(parent_id) = pli.get_provider_id() { + let category_id = pli.get_category_id().unwrap_or(0); + for episode in &mut episodes { + episode.header.virtual_id = target_id_mapping.get_and_update_virtual_id(&episode.header.uuid, provider_id, episode.header.item_type, parent_id); + episode.header.category_id = category_id; + provider_series.entry(pli.parent_code.clone()) + .or_default() + .push(ProviderEpisodeKey { + provider_id: episode.header.get_provider_id().unwrap_or(0), + virtual_id: episode.header.virtual_id, + }); + if target.use_memory_cache { + in_memory_updates.push( + VirtualIdRecord::new( + episode.header.get_provider_id().unwrap_or(0), + episode.header.virtual_id, + episode.header.item_type, + provider_id, + episode.get_uuid(), + ), + ); + } + } + } + } + + if !provider_series.is_empty() { + let mut series_pli = pli.clone(); + rewrite_provider_series_info_episode_virtual_id(&mut series_pli, &provider_series); + if let Err(err) = write_playlist_item_to_file(app_config, &target.name, &series_pli).await { + error!("Failed to persist series stream: {err}"); + } + app_state.playlists.update_playlist_items(target, vec![&series_pli]).await; + } + + if target.use_memory_cache && !in_memory_updates.is_empty() { + app_state.playlists.insert_playlist_items(target, episodes).await; + app_state.playlists.update_target_id_mapping(target, in_memory_updates).await; + } + } + } + } + } + } + } + } + } + + return Ok(content); } Err(str_to_io_error(&format!("Cant find stream with id: {}/{}/{}", @@ -202,7 +266,8 @@ pub async fn notify_account_expire(exp_date: Option, cfg: &Config, client: } } -pub async fn get_xtream_playlist(cfg: &Arc, client: &reqwest::Client, input: &Arc) -> (Vec, Vec) { +pub async fn download_xtream_playlist(cfg: &Arc, client: &reqwest::Client, input: &Arc) + -> (Vec, Vec) { let input_source: InputSource = { match input.staged.as_ref() { None => input.as_ref().into(), @@ -248,6 +313,8 @@ pub async fn get_xtream_playlist(cfg: &Arc, client: &reqwest::Client, in Ok(sub_playlist_parsed) => { if let Some(mut xtream_sub_playlist) = sub_playlist_parsed { playlist_groups.append(&mut xtream_sub_playlist); + } else { + error!("Could not parse playlist {xtream_cluster} for input {}: {}", input_source.name, sanitize_sensitive_info(&input_source.url)); } } Err(err) => errors.push(err) diff --git a/backend/src/utils/telegram.rs b/backend/src/utils/telegram.rs index 4755775fb..b178cd5dc 100644 --- a/backend/src/utils/telegram.rs +++ b/backend/src/utils/telegram.rs @@ -40,10 +40,10 @@ fn get_send_message_parse_mode_str(mode: &SendMessageParseMode) -> &'static str #[derive(Debug, serde::Serialize)] struct RequestObj { pub chat_id: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub message_thread_id: Option, pub text: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub parse_mode: Option, } diff --git a/config/config.yml b/config/config.yml index 00bee301a..ec994a980 100644 --- a/config/config.yml +++ b/config/config.yml @@ -113,16 +113,6 @@ library: formats: # - "nfo" # Optionally write Kodi-compatible NFO files - # Classification rules for movies vs series - classification: - # Regex patterns to identify series/TV shows - # If a file/folder name matches any pattern, it's classified as a series - series_patterns: - - "(?i)s\\d+e\\d+" # S01E02 format - - "(?i)\\d+x\\d+" # 1x02 format - - "(?i)season[\\s._-]*\\d+" # "Season 1" format - - "(?i)episode[\\s._-]*\\d+" # "Episode 1" format - # Playlist integration playlist: # Group name for movies in M3U/Xtream playlist diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 8e883c4dc..4fadefcdc 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -23,6 +23,8 @@ zeroize = "1" chrono = "0.4.42" bytes = "1" ciborium = "0.2.2" +hex = "0.4.3" +uuid = { version = "1.19.0", features = ["v4"] } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.82" diff --git a/shared/src/model/config/api_proxy.rs b/shared/src/model/config/api_proxy.rs index 32490396f..a6a462451 100644 --- a/shared/src/model/config/api_proxy.rs +++ b/shared/src/model/config/api_proxy.rs @@ -14,11 +14,11 @@ pub struct ApiProxyServerInfoDto { pub name: String, pub protocol: String, pub host: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub port: Option, pub timezone: String, pub message: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub path: Option, } diff --git a/shared/src/model/config/api_user.rs b/shared/src/model/config/api_user.rs index 42910a272..68a9a5d35 100644 --- a/shared/src/model/config/api_user.rs +++ b/shared/src/model/config/api_user.rs @@ -14,25 +14,25 @@ pub enum UserConnectionPermission { pub struct ProxyUserCredentialsDto { pub username: String, pub password: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub token: Option, #[serde(default = "ProxyType::default")] pub proxy: ProxyType, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub server: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub epg_timeshift: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub created_at: Option, - #[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")] + #[serde(default, deserialize_with = "deserialize_timestamp")] pub exp_date: Option, #[serde(default)] pub max_connections: u32, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub status: Option, #[serde(default = "default_as_true")] pub ui_enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub comment: Option, } diff --git a/shared/src/model/config/base.rs b/shared/src/model/config/base.rs index c67594b90..3d93968f8 100644 --- a/shared/src/model/config/base.rs +++ b/shared/src/model/config/base.rs @@ -12,25 +12,25 @@ pub struct ConfigDto { pub process_parallel: bool, pub api: ConfigApiDto, pub working_dir: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub backup_dir: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub user_config_dir: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub mapping_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub custom_stream_response_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub video: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub schedules: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub log: Option, #[serde(default)] pub user_access_control: bool, #[serde(default = "default_connect_timeout_secs")] pub connect_timeout_secs: u32, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub sleep_timer_mins: Option, #[serde(default)] pub update_on_boot: bool, @@ -40,17 +40,17 @@ pub struct ConfigDto { pub accept_insecure_ssl_certificates: bool, #[serde(default)] pub web_ui: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub messaging: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub reverse_proxy: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub hdhomerun: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub proxy: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub ipcheck: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub library: Option, } @@ -61,19 +61,19 @@ pub struct MainConfigDto { #[serde(default)] pub process_parallel: bool, pub working_dir: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub backup_dir: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub user_config_dir: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub mapping_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub custom_stream_response_path: Option, #[serde(default)] pub user_access_control: bool, #[serde(default = "default_connect_timeout_secs")] pub connect_timeout_secs: u32, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub sleep_timer_mins: Option, #[serde(default)] pub update_on_boot: bool, @@ -125,7 +125,7 @@ impl From<&ConfigDto> for MainConfigDto { // It has no other purpose than editing and saving the schedules #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] pub struct SchedulesConfigDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub schedules: Option>, } diff --git a/shared/src/model/config/cache.rs b/shared/src/model/config/cache.rs index 08a027829..113ec2600 100644 --- a/shared/src/model/config/cache.rs +++ b/shared/src/model/config/cache.rs @@ -10,15 +10,15 @@ use path_clean::PathClean; pub struct CacheConfigDto { #[serde(default)] pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub size: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub dir: Option, } impl CacheConfigDto { pub fn is_empty(&self) -> bool { - !self.enabled && is_blank_optional_string(&self.size) && is_blank_optional_string(&self.dir) + !self.enabled && is_blank_optional_string(self.size.as_ref()) && is_blank_optional_string(self.dir.as_ref()) } pub(crate) fn prepare(&mut self, working_dir: &str) -> Result<(), TuliproxError> { diff --git a/shared/src/model/config/epg.rs b/shared/src/model/config/epg.rs index c4d488c83..4b63f0233 100644 --- a/shared/src/model/config/epg.rs +++ b/shared/src/model/config/epg.rs @@ -27,9 +27,9 @@ impl EpgSourceDto { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Default)] #[serde(deny_unknown_fields)] pub struct EpgConfigDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub sources: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub smart_match: Option, #[serde(skip)] pub t_sources: Vec, diff --git a/shared/src/model/config/epg_smart_match.rs b/shared/src/model/config/epg_smart_match.rs index 1cc11adc8..149ee11ad 100644 --- a/shared/src/model/config/epg_smart_match.rs +++ b/shared/src/model/config/epg_smart_match.rs @@ -33,7 +33,7 @@ pub struct EpgSmartMatchConfigDto { #[serde(default)] pub enabled: bool, pub normalize_regex: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub strip: Option>, #[serde(default)] pub name_prefix: EpgNamePrefix, diff --git a/shared/src/model/config/hdhomerun.rs b/shared/src/model/config/hdhomerun.rs index e0cddcbbf..289d2d526 100644 --- a/shared/src/model/config/hdhomerun.rs +++ b/shared/src/model/config/hdhomerun.rs @@ -1,6 +1,6 @@ use crate::create_tuliprox_error_result; use crate::error::{TuliproxError, TuliproxErrorKind}; -use crate::utils::{default_as_true, generate_hdhr_device_id, generate_hdhr_device_id_from_base, hash_string, hex_encode, validate_hdhr_device_id}; +use crate::utils::{default_as_true, generate_hdhr_device_id, generate_hdhr_device_id_from_base, hash_string, validate_hdhr_device_id}; use log::warn; use std::collections::HashSet; @@ -82,12 +82,7 @@ impl HdHomeRunDeviceConfigDto { if self.device_udn == default_device_udn() || self.device_udn.is_empty() { let hash = hash_string(&self.name); // Format the hash into a valid UUID string - let p1 = hex_encode(&hash[0..4]); - let p2 = hex_encode(&hash[4..6]); - let p3 = hex_encode(&hash[6..8]); - let p4 = hex_encode(&hash[8..10]); - let p5 = hex_encode(&hash[10..16]); - self.device_udn = format!("{p1}-{p2}-{p3}-{p4}-{p5}"); + self.device_udn = hash.to_valid_uuid(); if include_computed { warn!("HDHomeRun device '{}' is missing a unique device_udn. A new one has been generated: {}", self.name, self.device_udn); } diff --git a/shared/src/model/config/input.rs b/shared/src/model/config/input.rs index 23183f149..bac23fa4d 100644 --- a/shared/src/model/config/input.rs +++ b/shared/src/model/config/input.rs @@ -200,9 +200,9 @@ impl ConfigInputOptionsDto { pub struct StagedInputDto { pub name: String, pub url: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub username: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub password: Option, #[serde(default)] pub method: InputFetchMethod, @@ -240,15 +240,15 @@ pub struct ConfigInputAliasDto { pub id: u16, pub name: String, pub url: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub username: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub password: Option, #[serde(default)] pub priority: i16, #[serde(default)] pub max_connections: u16, - #[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")] + #[serde(default, deserialize_with = "deserialize_timestamp")] pub exp_date: Option, } @@ -284,19 +284,19 @@ pub struct ConfigInputDto { pub headers: HashMap, #[serde(default)] pub url: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub epg: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub username: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub password: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub persist: Option, #[serde(default = "default_as_true")] pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub options: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub aliases: Option>, #[serde(default)] pub priority: i16, @@ -304,11 +304,11 @@ pub struct ConfigInputDto { pub max_connections: u16, #[serde(default)] pub method: InputFetchMethod, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub staged: Option, - #[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")] + #[serde(default, deserialize_with = "deserialize_timestamp")] pub exp_date: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub panel_api: Option, } @@ -341,9 +341,8 @@ impl ConfigInputDto { #[allow(clippy::cast_possible_truncation)] pub fn prepare(&mut self, index: u16, _include_computed: bool) -> Result { - let is_batch = matches!(self.input_type, InputType::M3uBatch | InputType::XtreamBatch); self.name = self.name.trim().to_owned(); - if self.name.is_empty() && !is_batch { + if self.name.is_empty() { return Err(info_err!("name for input is mandatory".to_owned())); } @@ -356,7 +355,7 @@ impl ConfigInputDto { } } - self.persist = get_trimmed_string(&self.persist); + self.persist = get_trimmed_string(self.persist.as_ref()); let mut current_index = index + 1; self.id = current_index; diff --git a/shared/src/model/config/ipcheck.rs b/shared/src/model/config/ipcheck.rs index da563a7bc..35d1a2abc 100644 --- a/shared/src/model/config/ipcheck.rs +++ b/shared/src/model/config/ipcheck.rs @@ -6,33 +6,33 @@ use regex::Regex; #[serde(deny_unknown_fields)] pub struct IpCheckConfigDto { /// URL that may return both IPv4 and IPv6 in one response - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub url: Option, /// Dedicated URL to fetch only IPv4 - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub url_ipv4: Option, /// Dedicated URL to fetch only IPv6 - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub url_ipv6: Option, /// Optional regex pattern to extract IPv4 - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub pattern_ipv4: Option, /// Optional regex pattern to extract IPv6 - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub pattern_ipv6: Option, } impl IpCheckConfigDto { pub fn is_empty(&self) -> bool { - is_blank_optional_string(&self.url) - && is_blank_optional_string(&self.url_ipv4) - && is_blank_optional_string(&self.url_ipv6) - && is_blank_optional_string(&self.pattern_ipv4) - && is_blank_optional_string(&self.pattern_ipv6) + is_blank_optional_string(self.url.as_ref()) + && is_blank_optional_string(self.url_ipv4.as_ref()) + && is_blank_optional_string(self.url_ipv6.as_ref()) + && is_blank_optional_string(self.pattern_ipv4.as_ref()) + && is_blank_optional_string(self.pattern_ipv6.as_ref()) } pub fn clean(&mut self) { diff --git a/shared/src/model/config/library.rs b/shared/src/model/config/library.rs index 4f5b27ac8..f22e9ec2d 100644 --- a/shared/src/model/config/library.rs +++ b/shared/src/model/config/library.rs @@ -5,7 +5,6 @@ use crate::utils::default_as_true; pub fn default_metadata_path() -> String { "library_metadata".to_string() } - fn default_tmdb_rate_limit_ms() -> u64 { 250 } @@ -24,15 +23,6 @@ fn default_movie_category() -> String { fn default_series_category() -> String { "Local TV Shows".to_string() } -fn default_series_patterns() -> Vec { - [ - r"S\d{2}E\d{2}", - r"s\d{2}e\d{2}", - r"\d{1,2}x\d{1,2}", - r"Season\s*\d+", - r"Episode\s*\d+", - ].iter().map(ToString::to_string).collect() -} fn default_supported_extensions() -> Vec { [ "mp4", @@ -57,8 +47,6 @@ pub struct LibraryConfigDto { #[serde(default)] pub metadata: LibraryMetadataConfigDto, #[serde(default)] - pub classification: LibraryClassificationConfigDto, - #[serde(default)] pub playlist: LibraryPlaylistConfigDto, } @@ -114,7 +102,7 @@ pub struct LibraryMetadataReadConfigDto { pub struct LibraryTmdbConfigDto { #[serde(default)] pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub api_key: Option, #[serde(default = "default_tmdb_rate_limit_ms")] pub rate_limit_ms: u64, @@ -130,15 +118,6 @@ pub enum LibraryMetadataFormat { Nfo, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct LibraryClassificationConfigDto { - #[serde(default = "default_series_patterns")] - pub series_patterns: Vec, - #[serde(default)] - pub series_directory_patterns: Vec, -} - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] #[serde(deny_unknown_fields)] pub struct LibraryPlaylistConfigDto { @@ -148,7 +127,6 @@ pub struct LibraryPlaylistConfigDto { pub series_category: String, } - impl LibraryConfigDto { pub fn prepare(&mut self) -> Result<(), TuliproxError> { // Validate enabled state diff --git a/shared/src/model/config/log.rs b/shared/src/model/config/log.rs index 3dd5dc20a..4a06e43af 100644 --- a/shared/src/model/config/log.rs +++ b/shared/src/model/config/log.rs @@ -7,7 +7,7 @@ pub struct LogConfigDto { pub sanitize_sensitive_info: bool, #[serde(default)] pub log_active_user: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub log_level: Option, } @@ -23,11 +23,11 @@ impl Default for LogConfigDto { impl LogConfigDto { pub fn is_empty(&self) -> bool { - self.sanitize_sensitive_info && !self.log_active_user && is_blank_optional_string(&self.log_level) + self.sanitize_sensitive_info && !self.log_active_user && is_blank_optional_string(self.log_level.as_ref()) } pub fn clean(&mut self) { - if is_blank_optional_string(&self.log_level) { + if is_blank_optional_string(self.log_level.as_ref()) { self.log_level = None; } } diff --git a/shared/src/model/config/macros.rs b/shared/src/model/config/macros.rs index 6a85fa602..c31f7a4fe 100644 --- a/shared/src/model/config/macros.rs +++ b/shared/src/model/config/macros.rs @@ -8,8 +8,8 @@ macro_rules! check_input_credentials { return Err(info_err!("url for input is mandatory".to_string())); } - $this.username = $crate::utils::get_trimmed_string(&$this.username); - $this.password = $crate::utils::get_trimmed_string(&$this.password); + $this.username = $crate::utils::get_trimmed_string($this.username.as_ref()); + $this.password = $crate::utils::get_trimmed_string($this.password.as_ref()); } match $input_type { InputType::M3u => { diff --git a/shared/src/model/config/messaging.rs b/shared/src/model/config/messaging.rs index ea077f618..af401bda4 100644 --- a/shared/src/model/config/messaging.rs +++ b/shared/src/model/config/messaging.rs @@ -38,7 +38,7 @@ pub struct PushoverMessagingConfigDto { impl PushoverMessagingConfigDto { pub fn is_empty(&self) -> bool { - is_blank_optional_string(&self.url) + is_blank_optional_string(self.url.as_ref()) && self.token.trim().is_empty() && self.user.trim().is_empty() } @@ -49,11 +49,11 @@ impl PushoverMessagingConfigDto { pub struct MessagingConfigDto { #[serde(default)] pub notify_on: Vec, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub telegram: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rest: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub pushover: Option, } diff --git a/shared/src/model/config/panel_api.rs b/shared/src/model/config/panel_api.rs index 52fdbd42d..265af4c39 100644 --- a/shared/src/model/config/panel_api.rs +++ b/shared/src/model/config/panel_api.rs @@ -24,7 +24,7 @@ pub struct PanelApiConfigDto { #[serde(default = "default_as_true")] pub enabled: bool, pub url: String, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub api_key: Option, #[serde(default)] pub query_parameter: PanelApiQueryParametersDto, diff --git a/shared/src/model/config/paths.rs b/shared/src/model/config/paths.rs index 06e28186a..e699cff33 100644 --- a/shared/src/model/config/paths.rs +++ b/shared/src/model/config/paths.rs @@ -5,6 +5,6 @@ pub struct ConfigPaths { pub sources_file_path: String, pub mapping_file_path: Option, pub api_proxy_file_path: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub custom_stream_response_path: Option, } diff --git a/shared/src/model/config/proxy.rs b/shared/src/model/config/proxy.rs index c9c086005..2e1c37e69 100644 --- a/shared/src/model/config/proxy.rs +++ b/shared/src/model/config/proxy.rs @@ -11,16 +11,16 @@ pub struct ProxyConfigDto { impl ProxyConfigDto { pub fn is_empty(&self) -> bool { - is_blank_optional_string(&self.username) - && is_blank_optional_string(&self.password) + is_blank_optional_string(self.username.as_ref()) + && is_blank_optional_string(self.password.as_ref()) && self.url.trim().is_empty() } pub fn clean(&mut self) { - if is_blank_optional_string(&self.username) { + if is_blank_optional_string(self.username.as_ref()) { self.username = None; } - if is_blank_optional_string(&self.password) { + if is_blank_optional_string(self.password.as_ref()) { self.password = None; } } diff --git a/shared/src/model/config/reverse_proxy.rs b/shared/src/model/config/reverse_proxy.rs index a31569df4..e730d1534 100644 --- a/shared/src/model/config/reverse_proxy.rs +++ b/shared/src/model/config/reverse_proxy.rs @@ -36,17 +36,17 @@ pub struct ReverseProxyConfigDto { #[serde(default)] pub resource_rewrite_disabled: bool, pub rewrite_secret: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub resource_retry: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub disabled_header: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub stream: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub cache: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rate_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub geoip: Option, } diff --git a/shared/src/model/config/schedule.rs b/shared/src/model/config/schedule.rs index 16be32d56..ec94fb3ec 100644 --- a/shared/src/model/config/schedule.rs +++ b/shared/src/model/config/schedule.rs @@ -3,6 +3,6 @@ pub struct ScheduleConfigDto { #[serde(default)] pub schedule: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub targets: Option>, } \ No newline at end of file diff --git a/shared/src/model/config/sort.rs b/shared/src/model/config/sort.rs index ffffa4af2..6f24a8262 100644 --- a/shared/src/model/config/sort.rs +++ b/shared/src/model/config/sort.rs @@ -42,7 +42,7 @@ impl Display for SortOrder { #[serde(deny_unknown_fields)] pub struct ConfigSortGroupDto { pub order: SortOrder, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub sequence: Option>, #[serde(skip)] pub t_sequence: Option>, @@ -85,7 +85,7 @@ pub struct ConfigSortChannelDto { // match against group title pub group_pattern: String, pub order: SortOrder, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub sequence: Option>, #[serde(skip)] pub t_sequence: Option>, @@ -136,9 +136,9 @@ impl ConfigSortChannelDto { pub struct ConfigSortDto { #[serde(default)] pub match_as_ascii: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub groups: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub channels: Option>, } diff --git a/shared/src/model/config/source.rs b/shared/src/model/config/source.rs index acb8530d4..03f1ab891 100644 --- a/shared/src/model/config/source.rs +++ b/shared/src/model/config/source.rs @@ -37,7 +37,7 @@ impl ConfigSourceDto { #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct SourcesConfigDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub templates: Option>, pub sources: Vec, } diff --git a/shared/src/model/config/stream.rs b/shared/src/model/config/stream.rs index beb6be6c9..764053bff 100644 --- a/shared/src/model/config/stream.rs +++ b/shared/src/model/config/stream.rs @@ -32,9 +32,9 @@ impl StreamBufferConfigDto { pub struct StreamConfigDto { #[serde(default)] pub retry: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub buffer: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub throttle: Option, #[serde(default = "default_grace_period_millis")] pub grace_period_millis: u64, diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index 3d8234a8f..c7dd4d97f 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -13,7 +13,7 @@ pub struct ConfigTargetOptions { pub share_live_streams: bool, #[serde(default)] pub remove_duplicates: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub force_redirect: Option, } @@ -44,9 +44,9 @@ pub struct XtreamTargetOutputDto { pub resolve_vod: bool, #[serde(default = "default_resolve_delay_secs")] pub resolve_vod_delay: u16, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub trakt: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub filter: Option, #[serde(skip)] pub t_filter: Option, @@ -94,13 +94,13 @@ impl XtreamTargetOutputDto { #[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct M3uTargetOutputDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub filename: Option, #[serde(default)] pub include_type_in_url: bool, #[serde(default)] pub mask_redirect_url: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub filter: Option, #[serde(skip)] pub t_filter: Option, @@ -127,7 +127,7 @@ impl M3uTargetOutputDto { #[serde(deny_unknown_fields)] pub struct StrmTargetOutputDto { pub directory: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub username: Option, #[serde(default)] pub style: StrmExportStyle, @@ -137,9 +137,9 @@ pub struct StrmTargetOutputDto { pub underscore_whitespace: bool, #[serde(default)] pub cleanup: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub strm_props: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub filter: Option, #[serde(skip)] pub t_filter: Option, @@ -161,7 +161,7 @@ impl StrmTargetOutputDto { pub struct HdHomeRunTargetOutputDto { pub device: String, pub username: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub use_output: Option, } @@ -204,22 +204,22 @@ pub struct ConfigTargetDto { pub enabled: bool, #[serde(default = "default_as_default")] pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub options: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub sort: Option, pub filter: String, #[serde(default)] pub output: Vec, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rename: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub mapping: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub favourites: Option>, #[serde(default)] pub processing_order: ProcessingOrder, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub watch: Option>, #[serde(default)] pub use_memory_cache: bool, diff --git a/shared/src/model/config/video_download.rs b/shared/src/model/config/video_download.rs index 921dd49bd..a15fbee6b 100644 --- a/shared/src/model/config/video_download.rs +++ b/shared/src/model/config/video_download.rs @@ -12,11 +12,11 @@ pub const DEFAULT_VIDEO_EXTENSIONS: [&str; 6] = ["mkv", "avi", "mp4", "mpeg", "d pub struct VideoDownloadConfigDto { #[serde(default)] pub headers: HashMap, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub directory: Option, #[serde(default)] pub organize_into_directories: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub episode_pattern: Option, } @@ -24,8 +24,8 @@ impl VideoDownloadConfigDto { pub fn is_empty(&self) -> bool { !self.organize_into_directories && self.headers.is_empty() - && is_blank_optional_string(&self.directory) - && is_blank_optional_string(&self.episode_pattern) + && is_blank_optional_string(self.directory.as_ref()) + && is_blank_optional_string(self.episode_pattern.as_ref()) } } @@ -34,16 +34,16 @@ impl VideoDownloadConfigDto { pub struct VideoConfigDto { #[serde(default)] pub extensions: Vec, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub download: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub web_search: Option, } impl VideoConfigDto { pub fn is_empty(&self) -> bool { - self.extensions.is_empty() && is_blank_optional_string(&self.web_search) + self.extensions.is_empty() && is_blank_optional_string(self.web_search.as_ref()) && (self.download.is_none() || self.download.as_ref().is_some_and(|d| d.is_empty())) } diff --git a/shared/src/model/config/web_auth.rs b/shared/src/model/config/web_auth.rs index 8ff5e3e2f..6b7e0d8ae 100644 --- a/shared/src/model/config/web_auth.rs +++ b/shared/src/model/config/web_auth.rs @@ -13,7 +13,7 @@ pub struct WebAuthConfigDto { pub secret: String, #[serde(default = "default_token_ttl_mins")] pub token_ttl_mins: u32, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub userfile: Option, } @@ -36,6 +36,6 @@ impl WebAuthConfigDto { && self.token_ttl_mins == empty.token_ttl_mins && self.issuer.trim().is_empty() && self.secret.trim().is_empty() - && is_blank_optional_string(&self.userfile) + && is_blank_optional_string(self.userfile.as_ref()) } } diff --git a/shared/src/model/config/web_ui.rs b/shared/src/model/config/web_ui.rs index 2413dbced..b92d11d3d 100644 --- a/shared/src/model/config/web_ui.rs +++ b/shared/src/model/config/web_ui.rs @@ -27,7 +27,7 @@ const RESERVED_PATHS: &[&str] = &[ pub struct ContentSecurityPolicyConfigDto { #[serde(default)] pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub custom_attributes: Option>, } @@ -74,13 +74,13 @@ pub struct WebUiConfigDto { pub enabled: bool, #[serde(default = "default_as_true")] pub user_ui_enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub content_security_policy: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub path: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub auth: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub player_server: Option, #[serde(default = "default_kick_secs")] pub kick_secs: u64, @@ -105,8 +105,8 @@ impl WebUiConfigDto { let empty = WebUiConfigDto::default(); self.enabled == empty.enabled && self.user_ui_enabled == empty.user_ui_enabled - && is_blank_optional_string(&self.path) - && is_blank_optional_string(&self.player_server) + && is_blank_optional_string(self.path.as_ref()) + && is_blank_optional_string(self.player_server.as_ref()) && self.kick_secs == default_kick_secs() && (self.content_security_policy.is_none() || self @@ -128,10 +128,10 @@ impl WebUiConfigDto { self.auth = None; } - if is_blank_optional_string(&self.path) { + if is_blank_optional_string(self.path.as_ref()) { self.path = None; } - if is_blank_optional_string(&self.player_server) { + if is_blank_optional_string(self.player_server.as_ref()) { self.player_server = None; } self.kick_secs = default_kick_secs(); diff --git a/shared/src/model/info_doc_utils.rs b/shared/src/model/info_doc_utils.rs new file mode 100644 index 000000000..b66dd3f36 --- /dev/null +++ b/shared/src/model/info_doc_utils.rs @@ -0,0 +1,66 @@ +use crate::model::xtream_const; +use serde_json::Value; + +pub struct InfoDocUtils {} + +impl InfoDocUtils { + pub fn extract_year_from_release_date(release_date: &str) -> Option { + if release_date.len() >= 4 { + release_date[..4].parse::().ok() + } else { + None + } + } + + pub fn make_bdpath_resource_url(resource_url: Option<&String>, bd_path: &str, index: usize, field_prefix: &str) -> String { + if let Some(url) = resource_url { + if bd_path.starts_with("http") { + return format!("{url}/{field_prefix}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH); + } + } + bd_path.to_string() + } + + pub fn limited(n: f64) -> String { + if n > 0.01 { + let s = n.to_string(); + if s.len() > 4 { + s[..4].to_string() + } else { + s + } + } else { + "0".to_string() + } + } + + pub fn as_string(value: Option<&String>) -> Value { + Value::String(value.map_or_else(String::new, |v| v.clone())) + } + + pub fn empty_string() -> Value { + Value::String(String::new()) + } + + pub fn as_value(value: Option<&String>) -> Value { + if let Some(text) = value { + if let Ok(result) = serde_json::from_str(text) { + return result; + } + } + Value::Array(Vec::new()) + } + + pub fn as_u32(value: u32) -> Value { + Value::Number(serde_json::Number::from(value)) + } + + pub fn make_resource_url(resource_url: Option<&String>, value: &str, field: &str) -> String { + if let Some(url) = resource_url { + if value.starts_with("http") { + return format!("{url}/{field}"); + } + } + value.to_string() + } +} \ No newline at end of file diff --git a/shared/src/model/ip_check.rs b/shared/src/model/ip_check.rs index c64f053bc..c697bed0c 100644 --- a/shared/src/model/ip_check.rs +++ b/shared/src/model/ip_check.rs @@ -1,8 +1,8 @@ #[derive(Debug, serde::Serialize, serde::Deserialize, Default)] pub struct IpCheckDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub ipv4: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub ipv6: Option, } \ No newline at end of file diff --git a/shared/src/model/mod.rs b/shared/src/model/mod.rs index 6f289b0f2..0828dc551 100644 --- a/shared/src/model/mod.rs +++ b/shared/src/model/mod.rs @@ -25,6 +25,7 @@ pub mod system_info; mod library_request; mod stream_properties; mod xtream; +mod info_doc_utils; pub use self::cluster_flags::*; pub use self::playlist::*; diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index ed86b70a8..b870496df 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -1,4 +1,5 @@ -use crate::model::{xtream_const, ClusterFlags, CommonPlaylistItem, ConfigTargetOptions, EpisodeStreamProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties}; +use crate::model::info_doc_utils::InfoDocUtils; +use crate::model::{xtream_const, ClusterFlags, CommonPlaylistItem, ConfigTargetOptions, EpisodeStreamProperties, LiveStreamProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties}; use crate::utils::{extract_extension_from_url, generate_playlist_uuid, get_provider_id}; use enum_iterator::Sequence; use serde::{Deserialize, Serialize}; @@ -7,12 +8,100 @@ use std::borrow::Cow; use std::fmt::Write; use std::fmt::{Display, Formatter}; use std::str::FromStr; +use hex::FromHex; +use uuid::Uuid; // https://de.wikipedia.org/wiki/M3U // https://siptv.eu/howto/playlist.html -pub type UUIDType = [u8; 32]; pub type VirtualId = u32; +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct UUIDType(pub [u8; 32]); + +impl UUIDType { + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Converts the first 16 bytes of this `UUIDType` into a valid UUID v4 string. + /// + /// Note: + /// - Only the first 16 bytes are used, because a standard UUID is 16 bytes. + /// - The remaining 16 bytes of the 32-byte `UUIDType` are ignored in this operation. + /// - This conversion is **not reversible**, calling `from_valid_uuid` on the resulting string + /// will not recover the original 32-byte `UUIDType`. + pub fn to_valid_uuid(&self) -> String { + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&self.0[0..16]); + + // Set UUID version (v4) + bytes[6] = (bytes[6] & 0x0F) | 0x40; + // Set UUID variant (10xxxxxx) + bytes[8] = (bytes[8] & 0x3F) | 0x80; + + format!( + "{}-{}-{}-{}-{}", + hex::encode_upper(&bytes[0..4]), + hex::encode_upper(&bytes[4..6]), + hex::encode_upper(&bytes[6..8]), + hex::encode_upper(&bytes[8..10]), + hex::encode_upper(&bytes[10..16]), + ) + } + + /// Creates a `UUIDType` from a valid UUID string. + /// + /// Implementation details: + /// - A standard UUID is 16 bytes. + /// - The first 16 bytes of the resulting `UUIDType` are taken from the parsed UUID. + /// - The remaining 16 bytes are filled by hashing the first 16 bytes using Blake3. + /// - This ensures the resulting `UUIDType` is 32 bytes, but this operation is **not reversible** + /// to the original 32-byte `UUIDType` if the input was previously generated with `to_valid_uuid`. + pub fn from_valid_uuid(uuid: &str) -> Self { + let bytes = if let Ok(parsed_uuid) = Uuid::from_str(uuid) { + let mut bytes = [0u8; 32]; + // die 16 UUID Bytes + bytes[..16].copy_from_slice(parsed_uuid.as_bytes()); + // die restlichen 16 Bytes = Hash der UUID (optional) + let hash = blake3::hash(parsed_uuid.as_bytes()); + bytes[16..].copy_from_slice(&hash.as_bytes()[..16]); + bytes + } else { + // falback + *blake3::hash(uuid.as_bytes()).as_bytes() + }; + + UUIDType(bytes) + } +} + +impl AsRef<[u8]> for UUIDType { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + + +impl FromStr for UUIDType { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + let bytes = <[u8; 32]>::from_hex(s)?; + Ok(UUIDType(bytes)) + } +} + +impl std::fmt::Display for UUIDType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", hex::encode(self.0)) + } +} + #[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default)] #[repr(u8)] pub enum XtreamCluster { @@ -195,6 +284,11 @@ pub trait PlaylistEntry: Send + Sync { fn get_provider_url(&self) -> String; fn get_uuid(&self) -> UUIDType; fn get_item_type(&self) -> PlaylistItemType; + fn get_group(&self) -> Cow<'_, str>; + fn get_name(&self) -> Cow<'_, str>; + fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option; + fn get_additional_properties(&self) -> Option<&StreamProperties>; + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties>; } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -429,6 +523,34 @@ impl PlaylistEntry for M3uPlaylistItem { fn get_item_type(&self) -> PlaylistItemType { self.item_type } + + #[inline] + fn get_group(&self) -> Cow<'_, str> { + Cow::Borrowed(self.group.as_str()) + } + + #[inline] + fn get_name(&self) -> Cow<'_, str> { + if self.title.is_empty() { + Cow::Borrowed(self.title.as_str()) + } else { + Cow::Borrowed(self.name.as_str()) + } + } + + #[inline] + fn get_resolved_info_document(&self, _options: &XtreamMappingOptions) -> Option { + None + } + #[inline] + fn get_additional_properties(&self) -> Option<&StreamProperties> { + None + } + #[inline] + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { + None + } + } macro_rules! generate_field_accessor_impl_for_m3u_playlist_item { @@ -472,9 +594,22 @@ pub struct XtreamMappingOptions { } impl XtreamMappingOptions { + #[inline] pub fn is_reverse(&self, item_type: PlaylistItemType) -> bool { self.reverse_item_types.is_set(item_type) } + + pub fn get_resource_url(&self, xtream_cluster: XtreamCluster, item_type: PlaylistItemType, virtual_id: VirtualId) -> Option { + let is_reverse = self.is_reverse(item_type); + let resource_url = if is_reverse && self.rewrite_resource_url && self.base_url.is_some() { + let resource_url = format!("{}/resource/{}/{}/{}/{}", self.base_url.as_ref().map_or_else(String::new, |b| b.clone()), + xtream_cluster.as_stream_type(), self.username, self.password, virtual_id); + Some(resource_url) + } else { + None + }; + resource_url + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -498,39 +633,7 @@ pub struct XtreamPlaylistItem { pub channel_no: u32, } -fn make_bdpath_resource_url(resource_url: Option<&String>, bd_path: &str, index: usize, field_prefix: &str) -> String { - if let Some(url) = resource_url { - if bd_path.starts_with("http") { - return format!("{url}/{field_prefix}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH); - } - } - bd_path.to_string() -} - -fn make_resource_url(resource_url: Option<&String>, value: &str, field: &str) -> String { - if let Some(url) = resource_url { - if value.starts_with("http") { - return format!("{url}/{field}"); - } - } - value.to_string() -} - - impl XtreamPlaylistItem { - pub fn get_container_extension(&self) -> Option { - match self.additional_properties { - None => None, - Some(ref props) => { - match props { - StreamProperties::Live(_) => Some("ts".to_string()), - StreamProperties::Video(video) => Some(video.container_extension.clone()), - StreamProperties::Series(_) => None, - StreamProperties::Episode(episode) => Some(episode.container_extension.clone()), - } - } - } - } pub fn to_common(&self) -> CommonPlaylistItem { CommonPlaylistItem { @@ -556,233 +659,323 @@ impl XtreamPlaylistItem { } } - pub fn to_document(&self, options: &XtreamMappingOptions) -> serde_json::Value { - let is_reverse = options.is_reverse(self.item_type); - let resource_url = if is_reverse && options.rewrite_resource_url && options.base_url.is_some() { - let resource_url = format!("{}/resource/{}/{}/{}/{}", options.base_url.as_ref().map_or_else(String::new, |b| b.clone()), - self.xtream_cluster.as_stream_type(), options.username, options.password, self.get_virtual_id()); - Some(resource_url) - } else { - None - }; - - if let Some(props) = self.additional_properties.as_ref() { - match props { - StreamProperties::Live(live) => { - let stream_icon = if !self.logo.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo, "logo") - } else if !self.logo_small.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") - } else { - String::new() - }; - Value::Object(serde_json::Map::from_iter([ - ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), - ("name".to_string(), Value::String(self.title.clone())), // name or title ? - ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), - ("stream_icon".to_string(), Value::String(stream_icon)), - ("epg_channel_id".to_string(), Value::String(self.epg_channel_id.as_ref().map_or_else(String::new, Clone::clone))), - ("added".to_string(), Value::String(String::new())), - ("is_adult".to_string(), Value::Number(serde_json::Number::from(live.is_adult))), - ("category_id".to_string(), Value::String(self.category_id.to_string())), - ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), - ("custom_sid".to_string(), live.custom_sid.as_ref().map_or(Value::Null, |s| Value::String(s.clone()))), - ("direct_source".to_string(), if options.skip_live_direct_source { Value::String(String::new()) } else { Value::String(live.direct_source.clone()) }), - ("tv_archive".to_string(), Value::Number(serde_json::Number::from(live.tv_archive.unwrap_or_default()))), - ("tv_archive_duration".to_string(), Value::Number(serde_json::Number::from(live.tv_archive_duration.unwrap_or_default()))), - ])) - } - StreamProperties::Video(video) => { - let stream_icon = if !self.logo.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo, "logo") - } else if !self.logo_small.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") - } else { - String::new() - }; - Value::Object(serde_json::Map::from_iter([ - ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), - ("name".to_string(), Value::String(self.title.clone())), // name or title ? - ("stream_type".to_string(), Value::String("movie".to_string())), - ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), - ("stream_icon".to_string(), Value::String(stream_icon)), - ("rating".to_string(), Value::String(video.rating.map_or_else(String::new, |r| format!("{r:.2}")))), - ("rating_5based".to_string(), Value::String(video.rating_5based.map_or_else(String::new, |r| format!("{r:.2}")))), - ("tmdb".to_string(), Value::String(video.tmdb.as_ref().map_or_else(String::new, ToString::to_string))), - ("trailer".to_string(), Value::String(video.trailer.as_ref().map_or_else(String::new, Clone::clone))), - ("added".to_string(), Value::String(video.added.clone())), - ("is_adult".to_string(), Value::Number(serde_json::Number::from(video.is_adult))), - ("category_id".to_string(), Value::String(self.category_id.to_string())), - ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), - ("container_extension".to_string(), Value::String(video.container_extension.clone())), - ("custom_sid".to_string(), video.custom_sid.as_ref().map_or(Value::Null, |s| Value::String(s.clone()))), - ("direct_source".to_string(), if options.skip_video_direct_source { Value::String(String::new()) } else { Value::String(video.direct_source.clone()) }), - ])) - } - StreamProperties::Series(series) => { - Value::Object(serde_json::Map::from_iter([ - ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), - ("name".to_string(), Value::String(self.title.clone())), // name or title ? - ("series_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), - ("cover".to_string(), Value::String(make_resource_url(resource_url.as_ref(), &series.cover, xtream_const::XC_PROP_COVER))), - ("plot".to_string(), Value::String(series.plot.as_ref().map_or_else(String::new, Clone::clone))), - ("cast".to_string(), Value::String(series.cast.clone())), - ("director".to_string(), Value::String(series.director.clone())), - ("genre".to_string(), Value::String(series.genre.as_ref().map_or_else(String::new, Clone::clone))), - ("release_date".to_string(), Value::String(series.release_date.as_ref().map_or_else(String::new, Clone::clone))), - ("releaseDate".to_string(), Value::String(series.release_date.as_ref().map_or_else(String::new, Clone::clone))), - ("last_modified".to_string(), Value::String(series.last_modified.as_ref().map_or_else(String::new, Clone::clone))), - ("rating".to_string(), Value::String(format!("{:.2}", series.rating))), - ("rating_5based".to_string(), Value::String(format!("{:.2}", series.rating_5based))), - ("backdrop_path".to_string(), Value::Array( - series.backdrop_path.as_ref().map_or_else(Vec::new, |b| b.iter().enumerate().map(|(idx, p)| - Value::String( - make_bdpath_resource_url(resource_url.as_ref(), p, idx, "") - ) - ).collect()) - )), - ("youtube_trailer".to_string(), Value::String(series.youtube_trailer.clone())), - ("tmdb".to_string(), Value::String(series.tmdb.as_ref().map_or_else(String::new, ToString::to_string))), - ("episode_runtime".to_string(), Value::String(series.episode_run_time.as_ref().map_or_else(String::new, Clone::clone))), - ("category_id".to_string(), Value::String(self.category_id.to_string())), - ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), - ])) - } - StreamProperties::Episode(_episode) => { - Value::Object(serde_json::Map::from_iter([])) - - // - // impl XtreamSeriesInfoEpisode { - // pub fn get_additional_properties(&self, series_info: &XtreamSeriesInfo) -> Option { - // let mut result = serde_json::Map::new(); - // let info = series_info.info.as_ref(); - // let bdpath = info.and_then(|i| i.backdrop_path.as_ref()); - // let bdpath_is_set = bdpath.as_ref().is_some_and(|bdpath| !bdpath.is_empty()); - // if bdpath_is_set { - // result.insert(String::from("backdrop_path"), Value::Array(Vec::from([Value::String(String::from(bdpath?.first()?))]))); - // } - // add_str_property_if_exists!(result, info.map_or("", |i| i.name.as_str()), "series_name"); - // add_str_property_if_exists!(result, info.map_or("", |i| get_non_empty_str(i.release_date.as_str(), i.releaseDate.as_str(), i.releasedate.as_str())), "series_release_date"); - // add_str_property_if_exists!(result, self.added.as_str(), "added"); - // add_str_property_if_exists!(result, info.map_or("", |i| i.cast.as_str()), "cast"); - // add_str_property_if_exists!(result, self.container_extension.as_str(), "container_extension"); - // add_str_property_if_exists!(result, self.info.as_ref().map_or("", |info| info.movie_image.as_str()), "cover"); - // add_str_property_if_exists!(result, info.map_or("", |i| i.director.as_str()), "director"); - // add_str_property_if_exists!(result, info.map_or("", |i| i.episode_run_time.as_str()), "episode_run_time"); - // add_str_property_if_exists!(result, info.map_or("", |i| i.last_modified.as_str()), "last_modified"); - // add_str_property_if_exists!(result, self.info.as_ref().map_or("", |info| info.plot.as_str()), "plot"); - // add_f64_property_if_exists!(result, info.map_or(0_f64, |i| i.rating), "rating"); - // add_f64_property_if_exists!(result, info.map_or(0_f64, |i| i.rating_5based), "rating_5based"); - // add_str_property_if_exists!(result, self.info.as_ref().map_or("", |info| get_non_empty_str(info.release_date.as_str(), info.releaseDate.as_str(), info.releasedate.as_str())), "release_date"); - // add_str_property_if_exists!(result, self.title, "title"); - // add_i64_property_if_exists!(result, self.season, "season"); - // add_i64_property_if_exists!(result, self.episode_num, "episode"); - // let series_tmdb_id = info.and_then(|i| i.tmdb_id.or(i.tmdb)); - // add_opt_i64_property_if_exists!(result, self.info.as_ref().and_then(|info| info.tmdb_id.or(info.tmdb.or(series_tmdb_id))), "tmdb_id"); - // - // // Add the "info" section to the playlist item additional properties. - // if let Some(episode_info) = &self.info { - // if let Ok(info_value) = serde_json::to_value(episode_info) { - // result.insert("info".to_string(), info_value); - // } - // } - // - // if result.is_empty() { - // None - // } else { - // serde_json::to_string(&Value::Object(result)).ok() - // } - // } - } - } - } else { - match self.xtream_cluster { - XtreamCluster::Live => { - let stream_icon = if !self.logo.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo, "logo") - } else if !self.logo_small.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") - } else { - String::new() - }; - Value::Object(serde_json::Map::from_iter([ - ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), - ("name".to_string(), Value::String(self.title.clone())), // name or title ? - ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), - ("stream_icon".to_string(), Value::String(stream_icon)), - ("epg_channel_id".to_string(), Value::String(self.epg_channel_id.as_ref().map_or_else(String::new, Clone::clone))), - ("added".to_string(), Value::String(String::new())), - ("is_adult".to_string(), Value::Number(serde_json::Number::from(0u32))), - ("category_id".to_string(), Value::String(self.category_id.to_string())), - ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), - ("custom_sid".to_string(), Value::Null), - ("direct_source".to_string(), Value::String(String::new())), - ("tv_archive".to_string(), Value::Number(serde_json::Number::from(0u32))), - ("tv_archive_duration".to_string(), Value::Number(serde_json::Number::from(0u32))), - ])) - } - XtreamCluster::Video => { - let stream_icon = if !self.logo.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo, "logo") - } else if !self.logo_small.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") - } else { - String::new() - }; - Value::Object(serde_json::Map::from_iter([ - ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), - ("name".to_string(), Value::String(self.title.clone())), // name or title ? - ("stream_type".to_string(), Value::String("movie".to_string())), - ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), - ("stream_icon".to_string(), Value::String(stream_icon)), - ("rating".to_string(), Value::String(String::new())), - ("rating_5based".to_string(), Value::String(String::new())), - ("tmdb".to_string(), Value::String(String::new())), - ("trailer".to_string(), Value::String(String::new())), - ("added".to_string(), Value::String(String::new())), - ("is_adult".to_string(), Value::Number(serde_json::Number::from(0u32))), - ("category_id".to_string(), Value::String(self.category_id.to_string())), - ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), - ("container_extension".to_string(), Value::String(String::new())), - ("custom_sid".to_string(), Value::Null), - ("direct_source".to_string(), Value::String(String::new())), - ])) - } - XtreamCluster::Series => { - let stream_icon = if !self.logo.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo, "logo") - } else if !self.logo_small.is_empty() { - make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") - } else { - String::new() - }; - Value::Object(serde_json::Map::from_iter([ - ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), - ("name".to_string(), Value::String(self.title.clone())), // name or title ? - ("series_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), - ("cover".to_string(), Value::String(stream_icon)), - ("plot".to_string(), Value::String(String::new())), - ("cast".to_string(), Value::String(String::new())), - ("director".to_string(), Value::String(String::new())), - ("genre".to_string(), Value::String(String::new())), - ("release_date".to_string(), Value::String(String::new())), - ("releaseDate".to_string(), Value::String(String::new())), - ("last_modified".to_string(), Value::String(String::new())), - ("rating".to_string(), Value::String(String::new())), - ("rating_5based".to_string(), Value::String(String::new())), - ("backdrop_path".to_string(), Value::Array(Vec::new())), - ("youtube_trailer".to_string(), Value::String(String::new())), - ("tmdb".to_string(), Value::String(String::new())), - ("episode_runtime".to_string(), Value::String(String::new())), - ("category_id".to_string(), Value::String(self.category_id.to_string())), - ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), - ])) + pub fn get_container_extension(&self) -> Option { + match self.additional_properties { + None => None, + Some(ref props) => { + match props { + StreamProperties::Live(_) => Some("ts".to_string()), + StreamProperties::Video(video) => Some(video.container_extension.clone()), + StreamProperties::Series(_) => None, + StreamProperties::Episode(episode) => Some(episode.container_extension.clone()), } } } } + #[inline] + pub fn has_details(&self) -> bool { + self.additional_properties.as_ref().is_some_and(|p| p.has_details()) + } + + pub fn to_info_document(&self, options: &XtreamMappingOptions) -> Value { + if self.has_details() { + if let Some(doc) = self.additional_properties.as_ref() + .map(|p| p.to_info_document(options, self.item_type, self.virtual_id, self.category_id)) { + return doc; + } + } + let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); + self.to_info_document_no_props(resource_url) + } + + fn to_info_document_no_props(&self, resource_url: Option) -> Value { + match self.xtream_cluster { + XtreamCluster::Live => { + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), InfoDocUtils::as_u32(self.channel_no)), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("stream_id".to_string(), InfoDocUtils::as_u32(self.virtual_id)), + ("stream_icon".to_string(), Value::String(stream_icon)), + ("epg_channel_id".to_string(), InfoDocUtils::as_string(self.epg_channel_id.as_ref())), + ("added".to_string(), InfoDocUtils::empty_string()), + ("is_adult".to_string(), InfoDocUtils::as_u32(0u32)), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ("custom_sid".to_string(), Value::Null), + ("direct_source".to_string(), InfoDocUtils::empty_string()), + ("tv_archive".to_string(), InfoDocUtils::as_u32(0u32)), + ("tv_archive_duration".to_string(), InfoDocUtils::as_u32(0u32)), + ])) + } + XtreamCluster::Video => { + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("stream_type".to_string(), Value::String("movie".to_string())), + ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("stream_icon".to_string(), Value::String(stream_icon)), + ("rating".to_string(), InfoDocUtils::empty_string()), + ("rating_5based".to_string(), InfoDocUtils::empty_string()), + ("tmdb".to_string(), InfoDocUtils::empty_string()), + ("trailer".to_string(), InfoDocUtils::empty_string()), + ("added".to_string(), InfoDocUtils::empty_string()), + ("is_adult".to_string(), Value::Number(serde_json::Number::from(0u32))), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ("container_extension".to_string(), InfoDocUtils::empty_string()), + ("custom_sid".to_string(), Value::Null), + ("direct_source".to_string(), InfoDocUtils::empty_string()), + ])) + } + XtreamCluster::Series => { + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("series_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("cover".to_string(), Value::String(stream_icon)), + ("plot".to_string(), InfoDocUtils::empty_string()), + ("cast".to_string(), InfoDocUtils::empty_string()), + ("director".to_string(), InfoDocUtils::empty_string()), + ("genre".to_string(), InfoDocUtils::empty_string()), + ("release_date".to_string(), InfoDocUtils::empty_string()), + ("releaseDate".to_string(), InfoDocUtils::empty_string()), + ("last_modified".to_string(), InfoDocUtils::empty_string()), + ("rating".to_string(), InfoDocUtils::empty_string()), + ("rating_5based".to_string(), InfoDocUtils::empty_string()), + ("backdrop_path".to_string(), Value::Array(Vec::new())), + ("youtube_trailer".to_string(), InfoDocUtils::empty_string()), + ("tmdb".to_string(), InfoDocUtils::empty_string()), + ("episode_runtime".to_string(), InfoDocUtils::empty_string()), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ])) + } + } + } + + pub fn to_document(&self, options: &XtreamMappingOptions) -> serde_json::Value { + if let Some(props) = self.additional_properties.as_ref() { + match props { + StreamProperties::Live(live) => { + self.live_to_document(options, live) + } + StreamProperties::Video(video) => { + self.video_to_document(options, video) + } + StreamProperties::Series(series) => { + self.series_to_document(options, series) + } + StreamProperties::Episode(_episode) => { + Value::Object(serde_json::Map::from_iter([])) + } + } + } else { + let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); + self.to_document_no_props(resource_url) + } + } + + fn series_to_document(&self, options: &XtreamMappingOptions, series: &SeriesStreamProperties) -> Value { + let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), + ("series_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("cover".to_string(), Value::String(InfoDocUtils::make_resource_url(resource_url.as_ref(), &series.cover, "cover"))), + ("plot".to_string(), InfoDocUtils::as_string(series.plot.as_ref())), + ("cast".to_string(), Value::String(series.cast.clone())), + ("director".to_string(), Value::String(series.director.clone())), + ("genre".to_string(), InfoDocUtils::as_string(series.genre.as_ref())), + ("release_date".to_string(), InfoDocUtils::as_string(series.release_date.as_ref())), + ("releaseDate".to_string(), InfoDocUtils::as_string(series.release_date.as_ref())), + ("last_modified".to_string(), InfoDocUtils::as_string(series.last_modified.as_ref())), + ("rating".to_string(), Value::String(InfoDocUtils::limited(series.rating))), + ("rating_5based".to_string(), Value::String(InfoDocUtils::limited(series.rating_5based))), + ("backdrop_path".to_string(), Value::Array( + series.backdrop_path.as_ref().map_or_else(Vec::new, |b| b.iter().enumerate().map(|(idx, p)| + Value::String( + InfoDocUtils::make_bdpath_resource_url(resource_url.as_ref(), p, idx, "") + ) + ).collect()) + )), + ("youtube_trailer".to_string(), Value::String(series.youtube_trailer.clone())), + ("tmdb".to_string(), Value::String(series.tmdb.as_ref().map_or_else(String::new, ToString::to_string))), + ("episode_runtime".to_string(), InfoDocUtils::as_string(series.episode_run_time.as_ref())), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ])) + } + + fn video_to_document(&self, options: &XtreamMappingOptions, video: &VideoStreamProperties) -> Value { + let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("stream_type".to_string(), Value::String(video.stream_type.as_ref().map_or_else(|| "movie".to_string(), Clone::clone))), + ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("stream_icon".to_string(), Value::String(stream_icon)), + ("rating".to_string(), Value::String(InfoDocUtils::limited(video.rating.unwrap_or_default()))), + ("rating_5based".to_string(), Value::Number(serde_json::Number::from_f64(video.rating_5based.unwrap_or_default()).unwrap_or_else(|| serde_json::Number::from(0)))), + ("tmdb".to_string(), Value::String(video.tmdb.as_ref().map_or_else(String::new, ToString::to_string))), + ("trailer".to_string(), InfoDocUtils::as_string(video.trailer.as_ref())), + ("added".to_string(), Value::String(video.added.clone())), + ("is_adult".to_string(), Value::Number(serde_json::Number::from(video.is_adult))), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ("container_extension".to_string(), Value::String(video.container_extension.clone())), + ("custom_sid".to_string(), video.custom_sid.as_ref().map_or(Value::Null, |s| Value::String(s.clone()))), + ("direct_source".to_string(), if options.skip_video_direct_source { InfoDocUtils::empty_string() } else { Value::String(video.direct_source.clone()) }), + ])) + } + + fn live_to_document(&self, options: &XtreamMappingOptions, live: &LiveStreamProperties) -> Value { + let resource_url = options.get_resource_url(self.xtream_cluster, self.item_type, self.virtual_id); + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("added".to_string(), InfoDocUtils::as_string(live.added.as_ref())), + ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("stream_icon".to_string(), Value::String(stream_icon)), + ("stream_type".to_string(), Value::String(live.stream_type.as_ref().map_or_else(|| "live".to_string(), Clone::clone))), + ("epg_channel_id".to_string(), InfoDocUtils::as_string(self.epg_channel_id.as_ref())), + ("added".to_string(), InfoDocUtils::empty_string()), + ("is_adult".to_string(), Value::Number(serde_json::Number::from(live.is_adult))), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ("custom_sid".to_string(), live.custom_sid.as_ref().map_or(Value::Null, |s| Value::String(s.clone()))), + ("direct_source".to_string(), if options.skip_live_direct_source { InfoDocUtils::empty_string() } else { Value::String(live.direct_source.clone()) }), + ("tv_archive".to_string(), Value::Number(serde_json::Number::from(live.tv_archive.unwrap_or_default()))), + ("tv_archive_duration".to_string(), Value::Number(serde_json::Number::from(live.tv_archive_duration.unwrap_or_default()))), + ])) + } + + fn to_document_no_props(&self, resource_url: Option) -> Value { + match self.xtream_cluster { + XtreamCluster::Live => { + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("stream_icon".to_string(), Value::String(stream_icon)), + ("epg_channel_id".to_string(), InfoDocUtils::as_string(self.epg_channel_id.as_ref())), + ("added".to_string(), InfoDocUtils::empty_string()), + ("is_adult".to_string(), Value::Number(serde_json::Number::from(0u32))), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ("custom_sid".to_string(), Value::Null), + ("direct_source".to_string(), InfoDocUtils::empty_string()), + ("tv_archive".to_string(), Value::Number(serde_json::Number::from(0u32))), + ("tv_archive_duration".to_string(), Value::Number(serde_json::Number::from(0u32))), + ])) + } + XtreamCluster::Video => { + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("stream_type".to_string(), Value::String("movie".to_string())), + ("stream_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("stream_icon".to_string(), Value::String(stream_icon)), + ("rating".to_string(), InfoDocUtils::empty_string()), + ("rating_5based".to_string(), InfoDocUtils::empty_string()), + ("tmdb".to_string(), InfoDocUtils::empty_string()), + ("trailer".to_string(), InfoDocUtils::empty_string()), + ("added".to_string(), InfoDocUtils::empty_string()), + ("is_adult".to_string(), Value::Number(serde_json::Number::from(0u32))), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ("container_extension".to_string(), InfoDocUtils::empty_string()), + ("custom_sid".to_string(), Value::Null), + ("direct_source".to_string(), InfoDocUtils::empty_string()), + ])) + } + XtreamCluster::Series => { + let stream_icon = if !self.logo.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo, "logo") + } else if !self.logo_small.is_empty() { + InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.logo_small, "logo_small") + } else { + String::new() + }; + Value::Object(serde_json::Map::from_iter([ + ("num".to_string(), Value::Number(serde_json::Number::from(self.channel_no))), + ("name".to_string(), Value::String(self.title.clone())), // name or title ? + ("series_id".to_string(), Value::Number(serde_json::Number::from(self.virtual_id))), + ("cover".to_string(), Value::String(stream_icon)), + ("plot".to_string(), InfoDocUtils::empty_string()), + ("cast".to_string(), InfoDocUtils::empty_string()), + ("director".to_string(), InfoDocUtils::empty_string()), + ("genre".to_string(), InfoDocUtils::empty_string()), + ("release_date".to_string(), InfoDocUtils::empty_string()), + ("releaseDate".to_string(), InfoDocUtils::empty_string()), + ("last_modified".to_string(), InfoDocUtils::empty_string()), + ("rating".to_string(), InfoDocUtils::empty_string()), + ("rating_5based".to_string(), InfoDocUtils::empty_string()), + ("backdrop_path".to_string(), Value::Array(Vec::new())), + ("youtube_trailer".to_string(), InfoDocUtils::empty_string()), + ("tmdb".to_string(), InfoDocUtils::empty_string()), + ("episode_runtime".to_string(), InfoDocUtils::empty_string()), + ("category_id".to_string(), Value::String(self.category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(self.category_id))]))), + ])) + } + } + } + + pub fn resolve_resource_url<'a>(&'a self, field: &str) -> Option> { + self.additional_properties.as_ref().and_then(|a| a.resolve_resource_url(field)) + } } + impl PlaylistEntry for XtreamPlaylistItem { #[inline] fn get_virtual_id(&self) -> VirtualId { @@ -809,6 +1002,37 @@ impl PlaylistEntry for XtreamPlaylistItem { fn get_item_type(&self) -> PlaylistItemType { self.item_type } + #[inline] + fn get_group(&self) -> Cow<'_, str> { + Cow::Borrowed(self.group.as_str()) + } + #[inline] + fn get_name(&self) -> Cow<'_, str> { + if self.title.is_empty() { + Cow::Borrowed(self.title.as_str()) + } else { + Cow::Borrowed(self.name.as_str()) + } + } + + fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option { + if self.has_details() { + self.additional_properties.as_ref() + .map(|p| p.to_info_document(options, self.get_item_type(), + self.get_virtual_id(), self.get_category_id().unwrap_or(0))) + } else { + None + } + } + + #[inline] + fn get_additional_properties(&self) -> Option<&StreamProperties> { + self.additional_properties.as_ref() + } + #[inline] + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { + self.additional_properties.as_mut() + } } pub fn get_backdrop_path_value<'a>(field: &'a str, value: Option<&'a Value>) -> Option> { @@ -865,22 +1089,30 @@ macro_rules! generate_field_accessor_impl_for_xtream_playlist_item { .map(|s| Cow::::Borrowed(s)) }) } else { - None + video.details.as_ref().and_then(|details| { + details.backdrop_path.as_ref().and_then(|p| p.first()) + .or(details.movie_image.as_ref()) + .or(details.cover_big.as_ref()) + .map(|s| Cow::::Borrowed(s)) + }) } } StreamProperties::Series(series) => { if field == xtream_const::XC_PROP_COVER { - series.backdrop_path.as_ref().and_then(|p| p.first()).map(|s| Cow::::Borrowed(s)) + if series.cover.is_empty() { + series.backdrop_path.as_ref().and_then(|p| p.first()).map(|s| Cow::::Borrowed(s)) + } else { + Some(Cow::Borrowed(&series.cover)) + } } else { - None + match series.backdrop_path.as_ref() { + None => if series.cover.is_empty() { None } else { Some(Cow::Borrowed(&series.cover)) } + Some(p) => p.first().map(|s| Cow::::Borrowed(s)) + } } } StreamProperties::Episode(episode) => { - if field == xtream_const::XC_PROP_COVER { - Some(Cow::::Borrowed(&episode.movie_image)) - } else { - None - } + Some(Cow::::Borrowed(&episode.movie_image)) } }, None => None, @@ -889,7 +1121,6 @@ macro_rules! generate_field_accessor_impl_for_xtream_playlist_item { None } } - } } } @@ -989,7 +1220,6 @@ impl PlaylistItem { } } - fn get_additional_properties(header: &PlaylistItemHeader) -> Option { match &header.additional_properties { Some(props) => Some(props.clone()), @@ -1009,7 +1239,7 @@ impl PlaylistItem { container_extension, rating: None, rating_5based: None, - stream_type: "movie".to_string(), + stream_type: Some("movie".to_string()), trailer: None, tmdb: None, is_adult: 0, @@ -1060,6 +1290,14 @@ impl PlaylistItem { } } } + + pub fn has_details(&self) -> bool { + self.header.additional_properties.as_ref().is_some_and(|p| p.has_details()) + } + + pub fn get_tmdb_id(&self) -> Option { + self.header.additional_properties.as_ref().and_then(|p| p.get_tmdb_id()) + } } impl PlaylistEntry for PlaylistItem { @@ -1092,6 +1330,38 @@ impl PlaylistEntry for PlaylistItem { fn get_item_type(&self) -> PlaylistItemType { self.header.item_type } + + #[inline] + fn get_group(&self) -> Cow<'_, str> { + Cow::Borrowed(self.header.group.as_str()) + } + + #[inline] + fn get_name(&self) -> Cow<'_, str> { + if self.header.title.is_empty() { + Cow::Borrowed(self.header.title.as_str()) + } else { + Cow::Borrowed(self.header.name.as_str()) + } + } + + fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option { + if self.has_details() { + self.header.additional_properties.as_ref().map(|p| + p.to_info_document(options, self.get_item_type(), self.get_virtual_id(), + self.get_category_id().unwrap_or(0))) + } else { + None + } + } + + fn get_additional_properties(&self) -> Option<&StreamProperties> { + self.header.additional_properties.as_ref() + } + #[inline] + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { + self.header.additional_properties.as_mut() + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/shared/src/model/playlist_categories.rs b/shared/src/model/playlist_categories.rs index 68d0fa1ab..c5e7b01e0 100644 --- a/shared/src/model/playlist_categories.rs +++ b/shared/src/model/playlist_categories.rs @@ -1,35 +1,35 @@ #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] pub struct PlaylistClusterCategoriesDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub live: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub vod: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub series: Option>, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] pub struct PlaylistCategoriesDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub xtream: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub m3u: Option, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] pub struct PlaylistClusterBouquetDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub live: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub vod: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub series: Option>, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] pub struct PlaylistBouquetDto { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub xtream: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub m3u: Option, } \ No newline at end of file diff --git a/shared/src/model/playlist_request.rs b/shared/src/model/playlist_request.rs index c76d113fa..a970846e4 100644 --- a/shared/src/model/playlist_request.rs +++ b/shared/src/model/playlist_request.rs @@ -40,13 +40,13 @@ pub struct CommonPlaylistItem { pub url: String, pub input_name: String, pub item_type: PlaylistItemType, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub epg_channel_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub xtream_cluster: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub additional_properties: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub category_id: Option, } @@ -79,22 +79,22 @@ impl From for UiPlaylistGroup { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] pub struct PlaylistCategoriesResponse { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub live: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub vod: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub series: Option>, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] pub struct UiPlaylistCategories { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub live: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub vod: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub series: Option>>, } @@ -109,7 +109,7 @@ impl From for UiPlaylistCategories { } fn filter_channels( - groups: &Option>>, + groups: Option<&Vec>>, text: &str, ) -> Option>> { // normalize search text (lowercase) @@ -149,7 +149,7 @@ fn filter_channels( }) } -fn filter_channels_re(groups: &Option>>, regex: &Regex) -> Option>> { +fn filter_channels_re(groups: Option<&Vec>>, regex: &Regex) -> Option>> { groups.as_ref().map(|gs| { gs.iter() .filter_map(|group| { @@ -199,16 +199,16 @@ impl UiPlaylistCategories { SearchRequest::Clear => None, SearchRequest::Text(text, _search_fields) => { let text_lc = text.to_lowercase(); - let live = filter_channels(&self.live, &text_lc); - let video = filter_channels(&self.vod, &text_lc); - let series = filter_channels(&self.series, &text_lc); + let live = filter_channels(self.live.as_ref(), &text_lc); + let video = filter_channels(self.vod.as_ref(), &text_lc); + let series = filter_channels(self.series.as_ref(), &text_lc); build_result(live, video, series) } SearchRequest::Regexp(text, _search_fields) => { if let Ok(regex) = Regex::new(text) { - let live = filter_channels_re(&self.live, ®ex); - let video = filter_channels_re(&self.vod, ®ex); - let series = filter_channels_re(&self.series, ®ex); + let live = filter_channels_re(self.live.as_ref(), ®ex); + let video = filter_channels_re(self.vod.as_ref(), ®ex); + let series = filter_channels_re(self.series.as_ref(), ®ex); build_result(live, video, series) } else { None diff --git a/shared/src/model/status_check.rs b/shared/src/model/status_check.rs index ff1e45225..e45c2c337 100644 --- a/shared/src/model/status_check.rs +++ b/shared/src/model/status_check.rs @@ -6,15 +6,15 @@ use crate::model::StreamInfo; pub struct StatusCheck { pub status: String, pub version: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub build_time: Option, pub server_time: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub cache: Option, pub active_users: usize, pub active_user_connections: usize, pub active_user_streams: Vec, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub active_provider_connections: Option>, } diff --git a/shared/src/model/stream_info.rs b/shared/src/model/stream_info.rs index 7167de0b0..dbdfff6d1 100644 --- a/shared/src/model/stream_info.rs +++ b/shared/src/model/stream_info.rs @@ -65,9 +65,9 @@ pub struct StreamInfo { pub user_agent: String, #[serde(default)] pub ts: u64, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub country: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub session_token: Option, } diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index aa5cd8b77..6eda479fe 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -1,10 +1,13 @@ -use crate::model::{PlaylistEntry, PlaylistItem, XtreamSeriesInfo, XtreamVideoInfo}; -use crate::utils::{deserialize_as_option_string, deserialize_as_string, deserialize_as_string_array, - deserialize_json_as_string, deserialize_number_from_string, +use crate::model::{ PlaylistEntry, PlaylistItemType, VirtualId, XtreamCluster, XtreamMappingOptions, XtreamSeriesInfo, XtreamVideoInfo}; +use crate::utils::{deserialize_as_option_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, string_default_on_null, + deserialize_as_string, deserialize_as_string_array, deserialize_json_as_string, string_or_number_u32}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; +use std::collections::HashMap; +use serde_json::Value; +use crate::model::info_doc_utils::InfoDocUtils; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct LiveStreamProperties { @@ -92,14 +95,14 @@ pub struct VideoStreamProperties { #[serde(default, deserialize_with = "deserialize_number_from_string")] pub rating_5based: Option, #[serde(default)] - pub stream_type: String, + pub stream_type: Option, #[serde(default)] pub trailer: Option, #[serde(default, deserialize_with = "deserialize_number_from_string")] pub tmdb: Option, #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] pub is_adult: i32, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub details: Option, } @@ -115,8 +118,8 @@ pub struct SeriesStreamDetailEpisodeProperties { pub title: String, #[serde(default, deserialize_with = "string_default_on_null")] pub container_extension: String, - #[serde(default, deserialize_with = "string_default_on_null")] - pub custom_sid: String, + #[serde(default)] + pub custom_sid: Option, #[serde(default, deserialize_with = "string_default_on_null")] pub added: String, #[serde(default, deserialize_with = "string_default_on_null")] @@ -183,7 +186,7 @@ pub struct SeriesStreamProperties { pub youtube_trailer: String, #[serde(default, deserialize_with = "deserialize_number_from_string")] pub tmdb: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub details: Option, } @@ -218,6 +221,15 @@ pub enum StreamProperties { } impl StreamProperties { + pub fn has_details(&self) -> bool { + match self { + StreamProperties::Video(video) => video.details.is_some(), + StreamProperties::Series(series) => series.details.is_some(), + StreamProperties::Live(_) + | StreamProperties::Episode(_) => false, + } + } + pub fn prepare(&mut self) { match self { StreamProperties::Live(live) => { @@ -282,7 +294,7 @@ impl StreamProperties { StreamProperties::Live(_) => None, StreamProperties::Video(video) => video.tmdb, StreamProperties::Series(series) => series.tmdb, - StreamProperties::Episode(epsiode) => epsiode.tmdb, + StreamProperties::Episode(episode) => episode.tmdb, } } @@ -358,10 +370,312 @@ impl StreamProperties { StreamProperties::Episode(episode) => episode.added.as_ref().and_then(|v| v.parse::().ok()), } } + + pub fn resolve_resource_url<'a>(&'a self, field: &str) -> Option> { + if field.starts_with("backdrop_path") { + if let StreamProperties::Series(series) = self { + if let Some(backdrop) = series.backdrop_path.as_ref() { + if let Some(url) = backdrop.first() { + return Some(Cow::Borrowed(url)); + } + } + } + return None; + } else if field.starts_with("nfo_backdrop_path") { + if let StreamProperties::Video(video) = self { + if let Some(details) = video.details.as_ref() { + if let Some(backdrop) = details.backdrop_path.as_ref() { + if let Some(url) = backdrop.first() { + return Some(Cow::Borrowed(url)); + } + } + } + } + return None; + } + + if field == "cover" { + if let StreamProperties::Series(series) = self { + return Some(Cow::Borrowed(series.cover.as_str())); + } + return None; + } + if field == "logo" || field == "logo_small" { + return match self { + StreamProperties::Live(live) => { + Some(Cow::Borrowed(live.stream_icon.as_str())) + } + StreamProperties::Video(video) => { + Some(Cow::Borrowed(video.stream_icon.as_str())) + } + StreamProperties::Series(series) => { + Some(Cow::Borrowed(series.cover.as_str())) + } + StreamProperties::Episode(episode) => { + Some(Cow::Borrowed(episode.movie_image.as_str())) + } + } + } + if field == "movie_image" { + if let StreamProperties::Episode(episode) = self { + return Some(Cow::Borrowed(episode.movie_image.as_str())) + } + return None; + } + if field == "nfo_cover_big" { + if let StreamProperties::Video(video) = self { + if let Some(details) = video.details.as_ref() { + if let Some(cover_big) = details.cover_big.as_ref() { + return Some(Cow::Borrowed(cover_big.as_str())); + } + } + } + } + + if field == "nfo_movie_image" { + if let StreamProperties::Video(video) = self { + if let Some(details) = video.details.as_ref() { + if let Some(movie_image) = details.movie_image.as_ref() { + return Some(Cow::Borrowed(movie_image.as_str())); + } + } + } + return None; + } + if field.starts_with("nfo_ep_") { + if let Some((season, episode_num, field)) = parse_season_episode_field(field) { + if let StreamProperties::Series(series) = self { + if let Some(details) = series.details.as_ref() { + if let Some(episodes) = details.episodes.as_ref() { + for episode in episodes { + if episode.season == season && episode_num == episode.episode_num && field == "movie_image" { + return Some(Cow::Borrowed(episode.movie_image.as_str())); + } + } + } + } + } + } + } + + None + } + + pub fn to_info_document(&self, options: &XtreamMappingOptions, item_type: PlaylistItemType, + virtual_id: VirtualId, category_id: u32) -> Value { + match self { + StreamProperties::Live(_live) => { + Value::Object(serde_json::Map::new()) + } + StreamProperties::Video(video) => { + self.video_to_info_document(options, video, item_type, virtual_id, category_id) + } + StreamProperties::Series(series) => { + self.series_to_info_document(options, series, item_type, virtual_id, category_id) + } + StreamProperties::Episode(_episode) => { + Value::Object(serde_json::Map::new()) + } + } + } + + fn series_to_info_document(&self, options: &XtreamMappingOptions, series: &SeriesStreamProperties, + item_type: PlaylistItemType, virtual_id: VirtualId, category_id: u32) -> Value { + let resource_url = options.get_resource_url(XtreamCluster::Series, item_type, virtual_id); + Value::Object(serde_json::Map::from_iter([ + ("seasons".to_string(), Value::Array(Vec::new())), + ("info".to_string(), Value::Object(serde_json::Map::from_iter([ + ("name".to_string(), Value::String(series.name.clone())), + ("cover".to_string(), Value::String(InfoDocUtils::make_resource_url(resource_url.as_ref(), series.cover.as_ref(), "cover"))), + ("plot".to_string(), InfoDocUtils::as_string(series.plot.as_ref())), + ("cast".to_string(), Value::String(series.cast.clone())), + ("director".to_string(), Value::String(series.director.clone())), + ("genre".to_string(), InfoDocUtils::as_string(series.genre.as_ref())), + ("release_date".to_string(), InfoDocUtils::as_string(series.release_date.as_ref())), + ("releaseDate".to_string(), InfoDocUtils::as_string(series.release_date.as_ref())), + ("last_modified".to_string(), InfoDocUtils::as_string(series.last_modified.as_ref())), + ("rating".to_string(), Value::String(InfoDocUtils::limited(series.rating))), + ("rating_5based".to_string(), Value::String(InfoDocUtils::limited(series.rating_5based))), + ("backdrop_path".to_string(), Value::Array( + series.backdrop_path.as_ref().map_or_else(Vec::new, |b| b.iter().enumerate().map(|(idx, p)| + Value::String( + InfoDocUtils::make_bdpath_resource_url(resource_url.as_ref(), p, idx, "") + ) + ).collect()) + )), + ("tmdb".to_string(), Value::String(series.tmdb.unwrap_or_default().to_string())), + ("youtube_trailer".to_string(), Value::String(series.youtube_trailer.clone())), + ("episode_run_time".to_string(), InfoDocUtils::as_string(series.episode_run_time.as_ref())), + ("category_id".to_string(), Value::String(category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([InfoDocUtils::as_u32(category_id)]))), + ]))), + if let Some(episodes) = series.details.as_ref().and_then(|d| d.episodes.as_ref()) { + ("episodes".to_string(), self.series_episodes_to_info_document(options, resource_url.as_ref(), episodes)) + } else { + ("episodes".to_string(), Value::Object(serde_json::Map::new())) + } + ])) + } + + fn video_to_info_document(&self, options: &XtreamMappingOptions, video: &VideoStreamProperties, + item_type: PlaylistItemType, virtual_id: VirtualId, category_id: u32) -> Value { + let resource_url = options.get_resource_url(XtreamCluster::Video, item_type, virtual_id); + let stream_icon = InfoDocUtils::make_resource_url(resource_url.as_ref(), &self.get_stream_icon(), "logo"); + + Value::Object(serde_json::Map::from_iter([ + if let Some(details) = video.details.as_ref() { + ("info".to_string(), Value::Object(serde_json::Map::from_iter([ + ("actors".to_string(), InfoDocUtils::as_string(details.actors.as_ref())), + ("age".to_string(), InfoDocUtils::as_string(details.age.as_ref())), + ("audio".to_string(), InfoDocUtils::as_value(details.audio.as_ref())), + ("video".to_string(), InfoDocUtils::as_value(details.video.as_ref())), + ("backdrop_path".to_string(), Value::Array( + details.backdrop_path.as_ref().map_or_else(Vec::new, |b| b.iter().enumerate().map(|(idx, p)| + Value::String( + InfoDocUtils::make_bdpath_resource_url(resource_url.as_ref(), p, idx, "nfo_") + ) + ).collect()) + )), + ("bitrate".to_string(), InfoDocUtils::as_u32(details.bitrate)), + ("cast".to_string(), InfoDocUtils::as_string(details.cast.as_ref())), + ("country".to_string(), InfoDocUtils::as_string(details.country.as_ref())), + ("cover_big".to_string(), Value::String(InfoDocUtils::make_resource_url(resource_url.as_ref(), &details.cover_big.as_ref().map_or_else(String::new, Clone::clone), "nfo_cover_big"))), + ("description".to_string(), InfoDocUtils::as_string(details.description.as_ref())), + ("director".to_string(), InfoDocUtils::as_string(details.director.as_ref())), + ("duration".to_string(), InfoDocUtils::as_string(details.duration.as_ref())), + ("duration_secs".to_string(), InfoDocUtils::as_string(details.duration_secs.as_ref())), + ("episode_run_time".to_string(), InfoDocUtils::as_u32(details.episode_run_time.unwrap_or_default())), + ("genre".to_string(), InfoDocUtils::as_string(details.genre.as_ref())), + ("kinopoisk_url".to_string(), InfoDocUtils::as_string(details.kinopoisk_url.as_ref())), + ("movie_image".to_string(), Value::String(InfoDocUtils::make_resource_url(resource_url.as_ref(), &details.cover_big.as_ref().map_or_else(String::new, Clone::clone), "nfo_movie_image"))), + ("mpaa_rating".to_string(), InfoDocUtils::as_string(details.mpaa_rating.as_ref())), + ("name".to_string(), Value::String(video.name.clone())), + ("o_name".to_string(), InfoDocUtils::as_string(details.o_name.as_ref())), + ("plot".to_string(), InfoDocUtils::as_string(details.plot.as_ref())), + ("rating".to_string(), Value::String(InfoDocUtils::limited(video.rating.unwrap_or_default()))), + ("rating_count_kinopoisk".to_string(), InfoDocUtils::as_u32(details.rating_count_kinopoisk)), + ("release_date".to_string(), InfoDocUtils::as_string(details.release_date.as_ref())), + ("runtime".to_string(), InfoDocUtils::as_string(details.runtime.as_ref())), + ("status".to_string(), InfoDocUtils::as_string(details.status.as_ref())), + ("tmdb_id".to_string(), Value::String(video.tmdb.unwrap_or_default().to_string())), + ("youtube_trailer".to_string(), InfoDocUtils::as_string(details.youtube_trailer.as_ref())), + ]))) + } else { + ("info".to_string(), Value::Object(serde_json::Map::from_iter([ + ("actors".to_string(), InfoDocUtils::empty_string()), + ("age".to_string(), InfoDocUtils::empty_string()), + ("audio".to_string(), InfoDocUtils::as_value(None)), + ("video".to_string(), InfoDocUtils::as_value(None)), + ("backdrop_path".to_string(), Value::Array(Vec::from_iter([ + Value::String(stream_icon.clone()), + ]))), + ("bitrate".to_string(), InfoDocUtils::as_u32(0)), + ("cast".to_string(), InfoDocUtils::empty_string()), + ("country".to_string(), InfoDocUtils::empty_string()), + ("cover_big".to_string(), Value::String(stream_icon.clone())), + ("description".to_string(), InfoDocUtils::empty_string()), + ("director".to_string(), InfoDocUtils::empty_string()), + ("duration".to_string(), Value::String("0".to_string())), + ("duration_secs".to_string(), Value::String("0".to_string())), + ("episode_run_time".to_string(), InfoDocUtils::as_u32(0)), + ("genre".to_string(), InfoDocUtils::empty_string()), + ("kinopoisk_url".to_string(), InfoDocUtils::empty_string()), + ("movie_image".to_string(), Value::String(stream_icon.clone())), + ("mpaa_rating".to_string(), InfoDocUtils::empty_string()), + ("name".to_string(), Value::String(video.name.clone())), + ("o_name".to_string(), Value::String(video.name.clone())), + ("plot".to_string(), InfoDocUtils::empty_string()), + ("rating".to_string(), Value::String(InfoDocUtils::limited(video.rating.unwrap_or_default()))), + ("rating_count_kinopoisk".to_string(), InfoDocUtils::as_u32(0)), + ("release_date".to_string(), InfoDocUtils::empty_string()), + ("runtime".to_string(), Value::String("0".to_string())), + ("status".to_string(), Value::String("Released".to_string())), + ("tmdb_id".to_string(), Value::String(video.tmdb.unwrap_or_default().to_string())), + ("youtube_trailer".to_string(), InfoDocUtils::empty_string()), + ]))) + }, + ("movie_data".to_string(), Value::Object(serde_json::Map::from_iter([ + ("added".to_string(), Value::String(video.added.clone())), + ("category_id".to_string(), Value::String(category_id.to_string())), + ("category_ids".to_string(), Value::Array(Vec::from([InfoDocUtils::as_u32(category_id)]))), + ("container_extension".to_string(), Value::String(video.container_extension.clone())), + ("custom_sid".to_string(), video.custom_sid.as_ref().map_or(Value::Null, |s| Value::String(s.clone()))), + ("direct_source".to_string(), if options.skip_video_direct_source { InfoDocUtils::empty_string() } else { Value::String(video.direct_source.clone()) }), + ("name".to_string(), Value::String(video.name.clone())), + ("stream_id".to_string(), Value::Number(serde_json::Number::from(virtual_id))), + ]))) + ])) + } + + fn series_episodes_to_info_document(&self, options: &XtreamMappingOptions, + resource_url: Option<&String>, + episodes: &Vec) -> Value { + let mut map: HashMap> = HashMap::new(); + for ep in episodes { + let value = Value::Object(serde_json::Map::from_iter([ + ("id".to_string(), InfoDocUtils::as_u32(ep.id)), + ("episode_num".to_string(), InfoDocUtils::as_u32(ep.episode_num)), + ("title".to_string(), Value::String(ep.title.clone())), + ("container_extension".to_string(), Value::String(ep.container_extension.clone())), + ("info".to_string(), Value::Object(serde_json::Map::from_iter([ + ("id".to_string(), InfoDocUtils::as_u32(ep.tmdb.unwrap_or_default())), + ("air_date".to_string(), Value::String(ep.release_date.clone())), + ("rating".to_string(), Value::Number(serde_json::Number::from_f64(ep.rating.unwrap_or_default()).unwrap_or_else(|| serde_json::Number::from(0)))), + ("movie_image".to_string(), Value::String(InfoDocUtils::make_resource_url(resource_url, &ep.movie_image, "nfo_movie_image"))), + ("duration".to_string(), Value::String(ep.duration.clone())), + ("duration_secs".to_string(), InfoDocUtils::as_u32(ep.duration_secs)), + ("video".to_string(), InfoDocUtils::as_value(ep.video.as_ref())), + ("audio".to_string(), InfoDocUtils::as_value(ep.audio.as_ref())), + ("bitrate".to_string(), InfoDocUtils::as_u32(ep.bitrate)), + ]))), + ("custom_sid".to_string(), ep.custom_sid.as_ref().map_or(Value::Null, |s| Value::String(s.clone()))), + ("added".to_string(), Value::String(ep.added.clone())), + ("season".to_string(), InfoDocUtils::as_u32(ep.season)), + ("direct_source".to_string(), if options.skip_series_direct_source { InfoDocUtils::empty_string() } else { Value::String(ep.direct_source.clone()) }), + ])); + map.entry(ep.season).or_default().push(value); + } + + Value::Object( + map.into_iter() + .map(|(season, episodes)| { + ( + season.to_string(), + Value::Array(episodes) + ) + }) + .collect::>() + ) + } + + } +fn parse_season_episode_field(s: &str) -> Option<(u32, u32, String)> { + let mut parts = s.split('_'); + + let (prefix, ep) = (parts.next()?, parts.next()?); + if prefix != "nfo" || ep != "ep" { + return None; + } + + let season: u32 = parts.next()?.parse().ok()?; + let episode: u32 = parts.next()?.parse().ok()?; + + let kind = parts.collect::>().join("_"); + if kind.is_empty() { + return None; + } + + Some((season, episode, kind)) +} + + impl VideoStreamProperties { - pub fn from_info(info: &XtreamVideoInfo, pli: &PlaylistItem) -> VideoStreamProperties { + pub fn from_info

(info: &XtreamVideoInfo, pli: &P) -> VideoStreamProperties + where + P: PlaylistEntry, + { let mut props = VideoStreamProperties { name: info.info.name.clone(), category_id: info.movie_data.category_id, @@ -373,7 +687,7 @@ impl VideoStreamProperties { container_extension: info.movie_data.container_extension.clone(), rating: None, // from PlaylistItem rating_5based: None, // from PlaylistItem - stream_type: "movie".to_string(), + stream_type: None, // from PlaylistItem trailer: info.info.youtube_trailer.clone(), tmdb: info.info.tmdb_id.parse::().ok(), is_adult: 0, // from PlaylistItem @@ -406,7 +720,7 @@ impl VideoStreamProperties { }), }; - if let Some(StreamProperties::Video(video)) = pli.header.additional_properties.as_ref() { + if let Some(StreamProperties::Video(video)) = pli.get_additional_properties() { props.rating = video.rating; props.rating_5based = video.rating_5based; props.stream_type = video.stream_type.clone(); @@ -417,6 +731,8 @@ impl VideoStreamProperties { props.trailer = video.trailer.clone(); } props.is_adult = video.is_adult; + } else { + props.stream_type = Some("movie".to_string()); } props @@ -424,7 +740,10 @@ impl VideoStreamProperties { } impl SeriesStreamProperties { - pub fn from_info(info: &XtreamSeriesInfo, pli: &PlaylistItem) -> SeriesStreamProperties { + pub fn from_info

(info: &XtreamSeriesInfo, pli: &P) -> SeriesStreamProperties + where + P: PlaylistEntry, + { SeriesStreamProperties { name: info.info.name.clone(), category_id: info.info.category_id, @@ -443,7 +762,7 @@ impl SeriesStreamProperties { youtube_trailer: info.info.youtube_trailer.clone(), tmdb: info.info.tmdb, details: Some(SeriesStreamDetailProperties { - year: extract_year_from_release_date(&info.info.release_date), + year: InfoDocUtils::extract_year_from_release_date(&info.info.release_date), episodes: info.episodes.as_ref().map(|list| list.iter().map(|e| { SeriesStreamDetailEpisodeProperties { @@ -489,12 +808,3 @@ impl EpisodeStreamProperties { } } } - -fn extract_year_from_release_date(release_date: &str) -> Option { - if release_date.len() >= 4 { - release_date[..4].parse::().ok() - } else { - None - } -} - diff --git a/shared/src/model/xtream.rs b/shared/src/model/xtream.rs index b8492cf88..16d270bfb 100644 --- a/shared/src/model/xtream.rs +++ b/shared/src/model/xtream.rs @@ -1,11 +1,11 @@ -use std::collections::BTreeMap; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde::ser::SerializeMap; -use serde_json::Value; use crate::utils::{deserialize_as_option_string, deserialize_as_string_array, - deserialize_number_from_string, deserialize_number_from_string_or_zero, - string_default_on_null, string_or_number_u32, - deserialize_json_as_string, deserialize_release_date}; + deserialize_json_as_string, deserialize_number_from_string, + deserialize_number_from_string_or_zero, deserialize_release_date, + string_default_on_null, string_or_number_u32}; +use serde::ser::SerializeMap; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; +use std::collections::BTreeMap; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct XtreamVideoInfoMovieData { @@ -165,8 +165,8 @@ pub struct XtreamSeriesInfoEpisode { pub container_extension: String, #[serde(default)] pub info: Option, - #[serde(default, deserialize_with = "string_default_on_null")] - pub custom_sid: String, + #[serde(default)] + pub custom_sid: Option, #[serde(default, deserialize_with = "string_default_on_null")] pub added: String, #[serde(default, deserialize_with = "string_or_number_u32")] @@ -197,14 +197,20 @@ where let mut result = Vec::new(); for inner in array { if let Some(inner_array) = inner.as_array() { + // Nested array case: [[ep1, ep2], [ep3]] for item in inner_array { let ep: XtreamSeriesInfoEpisode = serde_json::from_value(item.clone()) .map_err(serde::de::Error::custom)?; result.push(ep); } + } else if inner.is_object() { + // Flat array case: [ep1, ep2, ep3] + let ep: XtreamSeriesInfoEpisode = serde_json::from_value(inner.clone()) + .map_err(serde::de::Error::custom)?; + result.push(ep); } } - Ok(Some(result)) + Ok(if result.is_empty() { None } else { Some(result) }) } } Value::Object(object) => { diff --git a/shared/src/model/xtream_const.rs b/shared/src/model/xtream_const.rs index 461a828fd..e92280ab2 100644 --- a/shared/src/model/xtream_const.rs +++ b/shared/src/model/xtream_const.rs @@ -1,3 +1,5 @@ +use crate::model::XtreamCluster; + pub const XC_LIVE_ID: &str = "live_id"; pub const XC_VOO_ID: &str = "vod_id"; pub const XC_SERIES_ID: &str = "series_id"; @@ -19,9 +21,6 @@ pub const XC_TAG_ID: &str = "id"; pub const XC_TAG_CATEGORY_ID: &str = "category_id"; pub const XC_TAG_STREAM_ID: &str = "stream_id"; pub const XC_TAG_EPG_LISTINGS: &str = "epg_listings"; -pub const XC_INFO_RESOURCE_PREFIX: &str = "nfo_"; -pub const XC_INFO_RESOURCE_PREFIX_EPISODE: &str = "nfo_ep_"; -pub const XC_SEASON_RESOURCE_PREFIX: &str = "ssn_"; pub const XC_PROP_BACKDROP_PATH: &str = "backdrop_path"; pub const XC_PROP_COVER: &str = "cover"; pub const XC_TAG_CATEGORY_IDS: &str = "category_ids"; @@ -45,21 +44,21 @@ pub const XC_FILE_VOD_INFO: &str = "xtream_vod_info"; pub const XC_FILE_SERIES_EPISODE_RECORD: &str = "series_episode_record"; pub const XC_TAG_SERIES_INFO_LAST_MODIFIED: &str = "last_modified"; +pub const LIVE_STREAM_FIELDS: [&str;0] = []; -pub const LIVE_STREAM_FIELDS: &[&str] = &[]; - -pub const VIDEO_STREAM_FIELDS: &[&str] = &[ +pub const VIDEO_STREAM_FIELDS: [&str;14] = [ "release_date", "cast", "director", "episode_run_time", "genre", "stream_type", "title", "year", "youtube_trailer", "trailer", "plot", "rating_5based", "stream_icon", "container_extension" ]; -pub const SERIES_STREAM_FIELDS: &[&str] = &[ +pub const SERIES_STREAM_FIELDS: [&str;15] = [ XC_PROP_BACKDROP_PATH, "cast", XC_PROP_COVER, "director", "episode_run_time", "genre", "last_modified", "name", "plot", "rating_5based", "stream_type", "title", "year", "youtube_trailer", "trailer" ]; -pub const XTREAM_VOD_REWRITE_URL_PROPS: &[&str] = &[XC_PROP_COVER]; +pub const XTREAM_VOD_REWRITE_URL_PROPS: [&str; 1] = [XC_PROP_COVER]; +pub const XTREAM_CLUSTER: [XtreamCluster; 3] = [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series]; \ No newline at end of file diff --git a/shared/src/utils/hash_utils.rs b/shared/src/utils/hash_utils.rs index f7153049f..829a215ba 100644 --- a/shared/src/utils/hash_utils.rs +++ b/shared/src/utils/hash_utils.rs @@ -1,21 +1,11 @@ use base64::Engine; use base64::engine::general_purpose; -use std::fmt::Write; use url::Url; use crate::model::{PlaylistItemType, UUIDType}; -#[inline] -pub fn string_to_uuid_type(text: &str) -> UUIDType { - let mut bytes = [0u8; 32]; - let s_bytes = text.as_bytes(); - let len = s_bytes.len().min(32); - bytes[..len].copy_from_slice(&s_bytes[..len]); - bytes -} - #[inline] pub fn hash_bytes(bytes: &[u8]) -> UUIDType { - blake3::hash(bytes).into() + UUIDType(blake3::hash(bytes).into()) } /// generates a hash from a string @@ -29,13 +19,9 @@ pub fn short_hash(text: &str) -> String { hex_encode(&hash.as_bytes()[..8]) } - #[inline] pub fn hex_encode(bytes: &[u8]) -> String { - bytes.iter().fold(String::new(), |mut output, b| { - let _ = write!(output, "{b:02X}"); - output - }) + hex::encode_upper(bytes) } pub fn hex_decode(hex: &str) -> Result, String> { if !hex.len().is_multiple_of(2) { @@ -52,7 +38,7 @@ pub fn hex_decode(hex: &str) -> Result, String> { } pub fn hash_string_as_hex(url: &str) -> String { - hex_encode(&hash_string(url)) + hex_encode(hash_string(url).as_ref()) } pub fn extract_id_from_url(url: &str) -> Option { diff --git a/shared/src/utils/serde_utils.rs b/shared/src/utils/serde_utils.rs index 0fc1a1825..dca47afdc 100644 --- a/shared/src/utils/serde_utils.rs +++ b/shared/src/utils/serde_utils.rs @@ -167,7 +167,7 @@ where match deserialize_number_from_string(deserializer) { Ok(Some(v)) => Ok(v), Ok(None) => Ok(T::default()), - Err(e) => Err(serde::de::Error::custom(e)), + Err(e) => Err(e), } } diff --git a/shared/src/utils/string_utils.rs b/shared/src/utils/string_utils.rs index f71d15e6c..74807bbee 100644 --- a/shared/src/utils/string_utils.rs +++ b/shared/src/utils/string_utils.rs @@ -18,7 +18,7 @@ impl> Capitalize for T { } } -pub fn get_trimmed_string(value: &Option) -> Option { +pub fn get_trimmed_string(value: Option<&String>) -> Option { if let Some(v) = value { let trimmed = v.trim(); if !trimmed.is_empty() { @@ -58,7 +58,7 @@ pub fn get_non_empty_str<'a>(first: &'a str, second: &'a str, third: &'a str) -> } } -pub fn is_blank_optional_string(s: &Option) -> bool { +pub fn is_blank_optional_string(s: Option<&String>) -> bool { s.is_none() || s.as_ref().is_some_and(|s| s.trim().is_empty()) }