diff --git a/CHANGELOG.md b/CHANGELOG.md index c5b5e1899..2fa17fa58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - **Global Input Definitions**: To align input definitions with the SourceEditor, inputs are now defined globally in the `inputs` section of the config file. Each source can reference one or more inputs by their name in the `inputs` attribute. - **Data Format Migration**: Due to heavy refactoring, the old data format is invalid. You need to clean your `data` folder and update the playlists. - **B+Tree Storage Format**: Storage format has changed to a more efficient Slotted Page architecture. + - **Index optimization**: Added index to B+Tree to accelerate queries without tree traversal. + - **TargetIdMapping Optimization**: Refactored to use disk-based B+Tree operations, eliminating startup latency. + - **B+Tree Header Metadata**: Implemented efficient `BPlusTreeMetadata` Enum to persist `VirtualId` counter directly in the database header. + - **Fast Initialization**: `TargetIdMapping` now conditionally loads the tree, achieving near-instant startup for established databases. - **Configuration Renames**: - `threads` attribute in `config.yml` renamed to `process_parallel` (boolean). - Added mandatory `rewrite_secret` to `reverse_proxy` config for stable resource URLs. diff --git a/Cargo.lock b/Cargo.lock index 1d2de60bb..8655dcbfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1156,7 +1156,7 @@ dependencies = [ [[package]] name = "frontend" -version = "3.2.42" +version = "3.2.43" dependencies = [ "anyhow", "base64", @@ -1171,6 +1171,7 @@ dependencies = [ "gloo-timers 0.2.6", "gloo-utils 0.2.0", "implicit-clone", + "indexmap", "js-sys", "log", "prost", @@ -2526,6 +2527,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -3983,7 +3993,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.2.42" +version = "3.2.43" dependencies = [ "base64", "bitflags 2.10.0", @@ -4558,7 +4568,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.2.42" +version = "3.2.43" dependencies = [ "arc-swap", "async-compression", @@ -4590,6 +4600,7 @@ dependencies = [ "log", "lol_html", "lz4_flex", + "memmap2", "mime", "mime_guess", "notify", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d512b4b36..52a8588d4 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tuliprox" -version = "3.2.42" +version = "3.2.43" edition = "2021" rust-version = "1.89.0" @@ -59,6 +59,7 @@ base64 = "0.22" paste = "1.0" indexmap = "2" lz4_flex = "0.12" +memmap2 = "0.9" fs2 = "0.4" notify = "8.2" arc-swap = "1.8" diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index 34b0f0d8a..d30553c86 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -1,4 +1,3 @@ -use crate::BUILD_TIMESTAMP; use crate::api::endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext}; use crate::api::model::{create_channel_unavailable_stream, create_custom_video_stream_response, create_provider_connections_exhausted_stream, create_provider_stream, @@ -8,40 +7,44 @@ use crate::api::model::{create_channel_unavailable_stream, create_custom_video_s use crate::api::model::{tee_stream, UserSession}; use crate::api::model::{ProviderAllocation, ProviderConfig, ProviderStreamState, StreamDetails, StreamingStrategy}; use crate::api::panel_api::try_provision_account_on_exhausted; +use crate::auth::Fingerprint; use crate::model::{ConfigInput, ResourceRetryConfig}; use crate::model::{ConfigTarget, ProxyUserCredentials}; use crate::tools::lru_cache::LRUResourceCache; -use crate::utils::{request}; +use crate::utils::request::{content_type_from_ext, parse_range}; use crate::utils::{async_file_reader, async_file_writer, create_new_file_for_write, get_file_extension}; use crate::utils::{debug_if_enabled, trace_if_enabled}; -use crate::auth::Fingerprint; -use crate::utils::request::{content_type_from_ext, parse_range}; +use crate::utils::request; +use crate::BUILD_TIMESTAMP; use arc_swap::ArcSwapOption; +use axum::body::Body; use axum::http::{header, HeaderMap, HeaderValue, Response, StatusCode}; use axum::response::IntoResponse; +use bytes::Bytes; use chrono::{DateTime, Utc}; -use futures::{StreamExt, TryStreamExt}; +use futures::{stream, StreamExt, TryStreamExt}; use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation}; use log::{debug, error, info, log_enabled, trace, warn}; use reqwest::header::RETRY_AFTER; use serde::Serialize; -use shared::model::{Claims, InputFetchMethod, PlaylistEntry, PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, XtreamCluster}; -use shared::utils::{bin_serialize, default_grace_period_millis, human_readable_kbps, trim_slash}; +use shared::concat_string; +use shared::model::{Claims, InputFetchMethod, PlaylistEntry, PlaylistItemType, ProxyType, StreamChannel, TargetType, UserConnectionPermission, XtreamCluster}; +use shared::utils::{bin_serialize, default_grace_period_millis, human_readable_kbps, trim_slash, Internable}; use shared::utils::{ extract_extension_from_url, replace_url_extension, sanitize_sensitive_info, DASH_EXT, HLS_EXT, }; use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use std::convert::Infallible; use std::io::SeekFrom; use std::path::{Path, PathBuf}; -use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tokio::sync::Mutex; use tokio_util::io::ReaderStream; use url::Url; -use shared::concat_string; const CONTENT_TYPE_BIN: &str = "application/cbor"; @@ -68,6 +71,13 @@ macro_rules! try_option_bad_request { }; } +#[macro_export] +macro_rules! internal_server_error { + () => { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + }; +} + #[macro_export] macro_rules! try_unwrap_body { ($body:expr) => { @@ -126,6 +136,7 @@ pub use try_option_bad_request; pub use try_result_bad_request; pub use try_result_not_found; pub use try_unwrap_body; +pub use internal_server_error; pub fn get_server_time() -> String { chrono::offset::Local::now() @@ -171,7 +182,7 @@ pub async fn serve_file(file_path: &Path, mime_type: String) -> impl IntoRespons ) .body(body)) } - Err(_) => axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Err(_) => internal_server_error!(), } } @@ -279,10 +290,10 @@ pub fn get_stream_alternative_url( alias_input: &Arc, ) -> String { let Some(input_user_info) = input.get_user_info() else { - return stream_url.to_owned(); + return stream_url.to_string(); }; let Some(alt_input_user_info) = alias_input.get_user_info() else { - return stream_url.to_owned(); + return stream_url.to_string(); }; let modified = stream_url.replacen(&input_user_info.base_url, &alt_input_user_info.base_url, 1); @@ -290,11 +301,11 @@ pub fn get_stream_alternative_url( modified.replacen(&input_user_info.password, &alt_input_user_info.password, 1) } -async fn get_redirect_alternative_url<'a>( +async fn get_redirect_alternative_url( app_state: &Arc, - redirect_url: &'a str, + redirect_url: &Arc, input: &ConfigInput, -) -> Cow<'a, str> { +) -> Arc { if let Some((base_url, username, password)) = input.get_matched_config_by_url(redirect_url) { if let Some(provider_cfg) = app_state .active_provider @@ -309,15 +320,15 @@ async fn get_redirect_alternative_url<'a>( ) { new_url = new_url.replacen(old_username, new_username, 1); new_url = new_url.replacen(old_password, new_password, 1); - return Cow::Owned(new_url); + return new_url.into(); } // one has credentials the other not, something not right - return Cow::Borrowed(redirect_url); + return redirect_url.clone(); } - return Cow::Owned(new_url); + return new_url.into(); } } - Cow::Borrowed(redirect_url) + redirect_url.clone() } @@ -342,7 +353,7 @@ async fn resolve_streaming_strategy( stream_url: &str, fingerprint: &Fingerprint, input: &ConfigInput, - force_provider: Option<&str>, + force_provider: Option<&Arc>, ) -> StreamingStrategy { // allocate a provider connection let mut provider_connection_handle = match force_provider { @@ -385,9 +396,9 @@ async fn resolve_streaming_strategy( ProviderAllocation::Exhausted => { let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); ProviderStreamState::Custom(stream) - }, - ProviderAllocation::Available(_) => ProviderStreamState::Available(Some(selected_provider_name), url), - ProviderAllocation::GracePeriod(_) => ProviderStreamState::GracePeriod(Some(selected_provider_name), url), + } + ProviderAllocation::Available(_) => ProviderStreamState::Available(Some(selected_provider_name.intern()), url.intern()), + ProviderAllocation::GracePeriod(_) => ProviderStreamState::GracePeriod(Some(selected_provider_name.intern()), url.intern()), } } } @@ -468,7 +479,7 @@ async fn create_stream_response_details( item_type: PlaylistItemType, share_stream: bool, connection_permission: UserConnectionPermission, - force_provider: Option<&str>, + force_provider: Option<&Arc>, ) -> StreamDetails { let mut streaming_strategy = resolve_streaming_strategy(app_state, stream_url, fingerprint, input, force_provider).await; let config_grace_period_millis = app_state.app_config.config @@ -591,9 +602,9 @@ where let extension = self.stream_ext.map_or_else( || { extract_extension_from_url(url) - .map_or_else(String::new, std::string::ToString::to_string) + .map_or_else(String::new, ToString::to_string) }, - std::string::ToString::to_string, + ToString::to_string, ); // if there is an action_path (like for timeshift duration/start), it will be added in front of the stream_id @@ -624,13 +635,13 @@ where if params.target_type == TargetType::M3u { if redirect_request || is_dash_request { - let redirect_url = if is_hls_request { - Cow::Owned(replace_url_extension(&provider_url, HLS_EXT)) + let redirect_url: Arc = if is_hls_request { + replace_url_extension(&provider_url, HLS_EXT).into() } else { - provider_url + provider_url.clone() }; let redirect_url = if is_dash_request { - Cow::Owned(replace_url_extension(&redirect_url, DASH_EXT)) + replace_url_extension(&redirect_url, DASH_EXT).into() } else { redirect_url }; @@ -644,7 +655,7 @@ where } } else if params.target_type == TargetType::Xtream { let Some(provider_id) = params.provider_id else { - return Some(axum::http::StatusCode::BAD_REQUEST.into_response()); + return Some(StatusCode::BAD_REQUEST.into_response()); }; if redirect_request { @@ -673,7 +684,7 @@ where ) { None => { error!("Can't find stream url for target {target_name}, context {}, stream_id {virtual_id}", params.req_context); - return Some(axum::http::StatusCode::BAD_REQUEST.into_response()); + return Some(StatusCode::BAD_REQUEST.into_response()); } Some(url) => { match app_state @@ -684,7 +695,7 @@ where Some(provider_cfg) => { get_stream_alternative_url(&url, params.input, &provider_cfg) } - None => url, + None => url.to_string(), } } }; @@ -812,15 +823,15 @@ pub async fn force_provider_stream_response( if let (Some(stream), _stream_info) = create_channel_unavailable_stream( &app_state.app_config, &[], - axum::http::StatusCode::SERVICE_UNAVAILABLE, + StatusCode::SERVICE_UNAVAILABLE, ) { app_state.connection_manager.update_stream_detail(&fingerprint.addr, CustomVideoStreamType::ChannelUnavailable).await; debug!("Streaming custom stream"); try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::OK) + .status(StatusCode::OK) .body(axum::body::Body::from_stream(stream))) } else { - axum::http::StatusCode::BAD_REQUEST.into_response() + StatusCode::BAD_REQUEST.into_response() } } @@ -933,7 +944,7 @@ pub async fn stream_response( } try_unwrap_body!(response.body(axum::body::Body::from_stream(broadcast_stream))) } else { - axum::http::StatusCode::BAD_REQUEST.into_response() + StatusCode::BAD_REQUEST.into_response() } } else { // Previously, we would always check if the provider redirected the request. @@ -1010,7 +1021,7 @@ pub async fn stream_response( return stream_resp.into_response(); } app_state.connection_manager.release_provider_handle(stream_details.provider_handle).await; - axum::http::StatusCode::BAD_REQUEST.into_response() + StatusCode::BAD_REQUEST.into_response() } fn get_stream_throttle(app_state: &Arc) -> u64 { @@ -1048,7 +1059,7 @@ async fn try_shared_stream_response_if_any( { let (status_code, header_map) = get_stream_response_with_headers(Some(( headers.clone(), - axum::http::StatusCode::OK, + StatusCode::OK, ))); let mut stream_details = StreamDetails::from_stream(stream); @@ -1078,6 +1089,7 @@ pub async fn local_stream_response( _target: &ConfigTarget, _user: &ProxyUserCredentials, connection_permission: UserConnectionPermission, + check_path: bool, ) -> impl IntoResponse + Send { if log_enabled!(log::Level::Trace) { trace!("Try to open stream {}", sanitize_sensitive_info(&pli.url)); @@ -1110,19 +1122,19 @@ pub async fn local_stream_response( let path = match path.canonicalize() { Ok(canonical) => canonical, Err(err) => { - error!("Local library file path is corrupt {}: {err}", path.display()); + error!("Local file path is corrupt {}: {err}", path.display()); return StatusCode::NOT_FOUND.into_response(); } }; // Verify path is within allowed media directories // (requires configuration of allowed base paths) - if !is_path_within_allowed_directories(&path, &library_paths) { + if check_path && !is_path_within_allowed_directories(&path, &library_paths) { return StatusCode::FORBIDDEN.into_response(); } let Ok(mut file) = tokio::fs::File::open(&path).await else { return StatusCode::NOT_FOUND.into_response() }; - let Ok(metadata) = file.metadata().await else { return StatusCode::INTERNAL_SERVER_ERROR.into_response() }; + let Ok(metadata) = file.metadata().await else { return internal_server_error!() }; let file_size = metadata.len(); let range = req_headers @@ -1163,7 +1175,7 @@ pub async fn local_stream_response( if start > 0 { if let Err(_err) = file.seek(SeekFrom::Start(start)).await { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + return internal_server_error!(); } } @@ -1252,7 +1264,7 @@ fn get_add_cache_content( add_cache_content } -fn get_mime_type(headers: &axum::http::HeaderMap, resource_url: &str) -> Option { +fn get_mime_type(headers: &HeaderMap, resource_url: &str) -> Option { headers .get(header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) // Option<&str> @@ -1275,7 +1287,7 @@ async fn build_resource_stream_response( let mut response_builder = axum::response::Response::builder().status(status); let mime_type = get_mime_type(response.headers(), resource_url); - let has_content_range = response.headers().contains_key(axum::http::header::CONTENT_RANGE); + let has_content_range = response.headers().contains_key(header::CONTENT_RANGE); for (key, value) in response.headers() { let name = key.as_str(); let is_hop_by_hop = matches!( @@ -1297,7 +1309,7 @@ async fn build_resource_stream_response( .bytes_stream() .map_err(|err| StreamError::reqwest(&err)); // Cache only complete responses (200 OK without Content-Range) - let can_cache = status == axum::http::StatusCode::OK && !has_content_range; + let can_cache = status == StatusCode::OK && !has_content_range; if can_cache { debug!( "Caching eligible resource stream {sanitized_resource_url}"); let cache_resource_path = if let Some(cache) = app_state.cache.load().as_ref() { @@ -1437,7 +1449,7 @@ pub async fn resource_response( input: Option<&ConfigInput>, ) -> impl IntoResponse + Send { if resource_url.is_empty() { - return axum::http::StatusCode::NO_CONTENT.into_response(); + return StatusCode::NO_CONTENT.into_response(); } let filter: HeaderFilter = Some(Box::new(|key| { key != "if-none-match" && key != "if-modified-since" @@ -1462,10 +1474,10 @@ pub async fn resource_response( return resp; } // Upstream failure after retries - return axum::http::StatusCode::BAD_GATEWAY.into_response(); + return StatusCode::BAD_GATEWAY.into_response(); } error!("Url is malformed {}", sanitize_sensitive_info(resource_url)); - axum::http::StatusCode::BAD_REQUEST.into_response() + StatusCode::BAD_REQUEST.into_response() } pub fn separate_number_and_remainder(input: &str) -> (String, Option) { @@ -1482,8 +1494,8 @@ pub fn separate_number_and_remainder(input: &str) -> (String, Option) { /// # Panics pub fn empty_json_list_response() -> axum::response::Response { try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) .body("[]".to_owned())) } @@ -1511,8 +1523,8 @@ pub fn get_username_from_auth_header(token: &str, app_state: &Arc) -> /// # Panics pub fn redirect(url: &str) -> impl IntoResponse { try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::FOUND) - .header(axum::http::header::LOCATION, url) + .status(StatusCode::FOUND) + .header(header::LOCATION, url) .body(axum::body::Body::empty())) } @@ -1539,15 +1551,15 @@ pub async fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> pub fn bin_response(data: &T) -> impl IntoResponse + Send { match bin_serialize(data) { Ok(body) => ( - [(axum::http::header::CONTENT_TYPE, CONTENT_TYPE_BIN)], + [(header::CONTENT_TYPE, CONTENT_TYPE_BIN)], body, ).into_response(), - Err(_) => axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Err(_) => internal_server_error!(), } } pub fn json_response(data: &T) -> impl IntoResponse + Send { - (axum::http::StatusCode::OK, axum::Json(data)).into_response() + (StatusCode::OK, axum::Json(data)).into_response() } pub fn json_or_bin_response(accept: Option<&str>, data: &T) -> impl IntoResponse + Send { @@ -1557,8 +1569,117 @@ pub fn json_or_bin_response(accept: Option<&str>, data: &T) -> imp json_response(data).into_response() } -pub fn create_session_fingerprint(fingerprint: &str, username: &str, virtual_id: u32) -> String { - format!("{fingerprint}|{username}|{virtual_id}") +pub fn stream_json_or_bin_response

(accept: Option<&str>, data: Box + Send>) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, +{ + if accept.is_some_and(|a| a.contains(CONTENT_TYPE_BIN)) { + return stream_bin_array(data); + } + stream_json_array(data) +} + +pub fn create_session_fingerprint(fingerprint: &Fingerprint, username: &str, virtual_id: u32) -> String { + concat_string!(&fingerprint.key, "|", username, "|", &virtual_id.to_string()) +} + +pub fn stream_json_array

(iter: Box + Send>) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, +{ + let stream = stream::unfold( + (iter, true), + |(mut iter, first)| async move { + match iter.next() { + Some(item) => { + let mut json = String::new(); + if !first { + json.push(','); + } + let element = serde_json::to_string(&item).ok()?; + json.push_str(&element); + Some((Ok::(Bytes::from(json)), (iter, false))) + } + None => None, + } + }, + ); + + let body = Body::from_stream( + stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"[")) }) + .chain(stream) + .chain(stream::once(async { + Ok::<_, Infallible>(Bytes::from_static(b"]")) + })), + ); + + try_unwrap_body!(Response::builder() + .header(header::CONTENT_TYPE, "application/json") + .body(body)) +} + +pub fn stream_bin_array

(iter: Box + Send>) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, +{ + let stream = stream::unfold( + iter, + |mut iter| async move { + match iter.next() { + Some(item) => { + match bin_serialize(&item) { + Ok(buf) => Some((Ok::(Bytes::from(buf)), iter)), + Err(err) => { + warn!("CBOR serialization error in stream: {err}"); + Some((Ok::(Bytes::new()), iter)) // skip errors, continue + } + } + } + None => None, + } + }, + ); + + let body = Body::from_stream( + stream::once(async { + // CBOR: start indefinite-length array + Ok::<_, Infallible>(Bytes::from_static(&[0x9f])) + }) + .chain(stream) + .chain(stream::once(async { + // CBOR: end indefinite-length array + Ok::<_, Infallible>(Bytes::from_static(&[0xff])) + })), + ); + + try_unwrap_body!(Response::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE_BIN) + .body(body)) +} + +pub fn create_api_proxy_user(app_state: &Arc) -> ProxyUserCredentials { + let config = app_state.app_config.config.load(); + + let server = config + .web_ui + .as_ref() + .and_then(|web_ui| web_ui.player_server.as_ref()) + .map_or("default", |server_name| server_name.as_str()); + + ProxyUserCredentials { + username: "api_user".to_string(), + password: "api_user".to_string(), + token: None, + proxy: ProxyType::Reverse(None), + server: Some(server.to_string()), + epg_timeshift: None, + created_at: None, + exp_date: None, + max_connections: 0, + status: None, + ui_enabled: false, + comment: None, + } } #[cfg(test)] diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index cb1d1031e..bfefbb1df 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -1,150 +1,53 @@ use crate::model::{AppConfig, ConfigInput, ConfigTarget}; use crate::repository::{m3u_repository, xtream_repository}; use crate::utils::{m3u, xtream}; -use crate::utils; use axum::response::IntoResponse; -use indexmap::IndexMap; -use serde::Serialize; use serde_json::{json}; -use shared::model::{CommonPlaylistItem, InputType, M3uPlaylistItem, PlaylistCategoriesResponse, - PlaylistGroup, PlaylistItemType, PlaylistResponseGroup, TargetType, XtreamCluster}; +use shared::model::{CommonPlaylistItem, InputType, TargetType, XtreamCluster}; use std::sync::Arc; -use crate::api::api_utils::{json_or_bin_response}; +use crate::api::api_utils::{empty_json_list_response, json_or_bin_response, stream_json_or_bin_response}; +use shared::utils::interner_gc; -fn group_playlist_items( - cluster: XtreamCluster, - iter: impl Iterator, - get_group: fn(&T) -> Arc, -) -> Vec -where - T: Serialize + Into, -{ - let mut groups: IndexMap, Vec> = IndexMap::new(); - - for item in iter { - let group_key = get_group(&item); - groups.entry(group_key) - .or_default() - .push(item); - } - - groups - .into_iter() - .enumerate() - .map(|(index, (key, value))| PlaylistResponseGroup { - #[allow(clippy::cast_possible_truncation)] - id: index as u32, - title: key.clone(), - channels: value.into_iter().map(Into::into).collect(), - xtream_cluster: cluster, - }) - .collect() -} - -fn group_playlist_items_by_cluster(params: Option<(utils::FileReadGuard, - impl Iterator)>) -> - (Vec, Vec, Vec) { - match params { - None => (vec![], vec![], vec![]), - Some((guard, iter)) => { - let mut live = Vec::new(); - let mut video = Vec::new(); - let mut series = Vec::new(); - for (item, _) in iter { - match item.item_type { - PlaylistItemType::Live - | PlaylistItemType::LiveUnknown - | PlaylistItemType::LiveHls - | PlaylistItemType::LiveDash => { - live.push(item); - } - PlaylistItemType::Catchup - | PlaylistItemType::Video - | PlaylistItemType::LocalVideo => { - video.push(item); - } - PlaylistItemType::Series - | PlaylistItemType::SeriesInfo - | PlaylistItemType::LocalSeries - | PlaylistItemType::LocalSeriesInfo => { - series.push(item); - } - } - } - - drop(guard); - - (live, video, series) - } - } -} - -fn group_playlist_groups_by_cluster(playlist: Vec) -> (Vec, Vec, Vec) { - let mut live = Vec::new(); - let mut video = Vec::new(); - let mut series = Vec::new(); - for group in playlist { - let channels = group.channels.iter() - .map(CommonPlaylistItem::from) - .collect(); - let grp = PlaylistResponseGroup { - id: group.id, - title: group.title, - channels, - xtream_cluster: group.xtream_cluster, - }; - match group.xtream_cluster { - XtreamCluster::Live => live.push(grp), - XtreamCluster::Video => video.push(grp), - XtreamCluster::Series => series.push(grp), - } - } - (live, video, series) -} - -async fn grouped_channels( - cfg: &AppConfig, - target: &ConfigTarget, - cluster: XtreamCluster, -) -> Option> { - xtream_repository::iter_raw_xtream_playlist(cfg, target, cluster).await - .map(|(_guard, iter)| group_playlist_items::( - cluster, - iter.filter(|(item, _)| item.item_type != PlaylistItemType::LocalSeries).map(|(v, _)| v.to_common()), - |item| item.group.clone(), - )) -} - -pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&str>) -> impl IntoResponse + Send { +pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, cluster: XtreamCluster, accept: Option<&str>) -> impl IntoResponse + Send { if let Some(target) = cfg_target { if target.has_output(TargetType::Xtream) { - let live_channels = grouped_channels(cfg, target, XtreamCluster::Live).await; - let vod_channels = grouped_channels(cfg, target, XtreamCluster::Video).await; - let series_channels = grouped_channels(cfg, target, XtreamCluster::Series).await; - - let response = PlaylistCategoriesResponse { - live: live_channels, - vod: vod_channels, - series: series_channels, + let Some((_guard, channel_iterator)) = xtream_repository::iter_raw_xtream_target_playlist(cfg, target, cluster).await else { + return empty_json_list_response(); }; - - return json_or_bin_response(accept, &response).into_response(); + let converted_iterator: Box + Send> = Box::new(channel_iterator.map(CommonPlaylistItem::from)); + return stream_json_or_bin_response(accept, converted_iterator).into_response(); } else if target.has_output(TargetType::M3u) { - let all_channels = m3u_repository::iter_raw_m3u_playlist(cfg, target).await; - let (live_channels, vod_channels, series_channels) = group_playlist_items_by_cluster(all_channels); - let response = PlaylistCategoriesResponse { - live: Some(group_playlist_items::(XtreamCluster::Live, live_channels.into_iter(), |item| item.group.clone())), - vod: Some(group_playlist_items::(XtreamCluster::Video, vod_channels.into_iter(), |item| item.group.clone())), - series: Some(group_playlist_items::(XtreamCluster::Series, series_channels.into_iter(), |item| item.group.clone())), + let Some((_guard, channel_iterator)) = m3u_repository::iter_raw_m3u_target_playlist(cfg, target, Some(cluster)).await else { + return empty_json_list_response(); }; - - return json_or_bin_response(accept, &response).into_response(); + let converted_iterator: Box + Send> = Box::new(channel_iterator.map(CommonPlaylistItem::from)); + return stream_json_or_bin_response(accept, converted_iterator).into_response(); } } (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response() } -pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cfg_input: Option<&Arc>, app_config: &Arc, accept: Option<&str>) -> impl IntoResponse + Send { + +pub(in crate::api::endpoints) async fn get_playlist_for_input(cfg_input: Option<&Arc>, cfg: &AppConfig, cluster: XtreamCluster, accept: Option<&str>) -> impl IntoResponse + Send { + if let Some(input) = cfg_input { + if matches!(input.input_type, InputType::Xtream | InputType::XtreamBatch) { + let Some((_guard, channel_iterator)) = xtream_repository::iter_raw_xtream_input_playlist(cfg, input, cluster).await else { + return empty_json_list_response(); + }; + let converted_iterator: Box + Send> = Box::new(channel_iterator.map(CommonPlaylistItem::from)); + return stream_json_or_bin_response(accept, converted_iterator).into_response(); + } else if matches!(input.input_type, InputType::M3u | InputType::M3uBatch) { + let Some((_guard, channels)) = m3u_repository::iter_raw_m3u_input_playlist(cfg, input, Some(cluster)).await else { + return empty_json_list_response(); + }; + let converted_iterator: Box + Send> = Box::new(channels.map(CommonPlaylistItem::from)); + return stream_json_or_bin_response(accept, converted_iterator).into_response(); + } + } + (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response() +} + +pub(in crate::api::endpoints) async fn get_playlist_for_custom_provider(client: &reqwest::Client, cfg_input: Option<&Arc>, app_config: &Arc, cluster: XtreamCluster, accept: Option<&str>) -> impl IntoResponse + Send { let cfg = app_config.config.load(); match cfg_input { Some(input) => { @@ -152,7 +55,7 @@ pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cf match input.input_type { InputType::M3u | InputType::M3uBatch => m3u::download_m3u_playlist(client, &cfg, input).await, InputType::Xtream | InputType::XtreamBatch => { - let (pl, err, _) = xtream::download_xtream_playlist(app_config, client, input, None).await; + let (pl, err, _) = xtream::download_xtream_playlist(app_config, client, input, Some(&[cluster])).await; (pl, err) } InputType::Library => { @@ -160,16 +63,12 @@ pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cf } }; if result.is_empty() { - let error_strings: Vec = errors.iter().map(std::string::ToString::to_string).collect(); + let error_strings: Vec = errors.iter().map(ToString::to_string).collect(); (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": error_strings.join(", ")}))).into_response() } else { - let (live, vod, series) = group_playlist_groups_by_cluster(result); - let response = PlaylistCategoriesResponse { - live: Some(live), - vod: Some(vod), - series: Some(series), - }; - json_or_bin_response(accept, &response).into_response() + let channels: Vec = result.iter().flat_map(|g| g.channels.iter()).map(CommonPlaylistItem::from).collect(); + interner_gc(); + json_or_bin_response(accept, &channels).into_response() } } None => (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response(), diff --git a/backend/src/api/endpoints/hdhomerun_api.rs b/backend/src/api/endpoints/hdhomerun_api.rs index 0b3513dbd..c1fe49433 100644 --- a/backend/src/api/endpoints/hdhomerun_api.rs +++ b/backend/src/api/endpoints/hdhomerun_api.rs @@ -1,7 +1,8 @@ -use crate::api::api_utils::try_unwrap_body; +use crate::api::api_utils::{try_unwrap_body, internal_server_error}; use crate::api::model::HdHomerunAppState; use crate::auth::AuthBasic; use crate::model::{AppConfig, ConfigTarget, ProxyUserCredentials}; +use crate::utils::arc_str_serde; use crate::processing::parser::xtream::get_xtream_url; use crate::repository::m3u_playlist_iterator::M3uPlaylistIterator; use crate::repository::m3u_repository; @@ -20,12 +21,12 @@ use std::sync::Arc; #[derive(Serialize, Deserialize, Clone)] struct Lineup { - #[serde(rename = "GuideNumber")] - guide_number: String, - #[serde(rename = "GuideName")] - guide_name: String, - #[serde(rename = "URL")] - url: String, + #[serde(with = "arc_str_serde", rename = "GuideNumber")] + guide_number: Arc, + #[serde(with = "arc_str_serde", rename = "GuideName")] + guide_name: Arc, + #[serde(with = "arc_str_serde", rename = "URL")] + url: Arc, } #[derive(Serialize, Deserialize, Clone)] @@ -124,13 +125,13 @@ where container_extension.as_deref(), live_stream_use_prefix, live_stream_without_extension, - ), + ).into(), }; let lineup = Lineup { - guide_number: item.epg_channel_id.unwrap_or(item.name).clone(), + guide_number: item.epg_channel_id.unwrap_or(item.name.clone()), guide_name: item.title.clone(), - url: stream_url, + url: stream_url.clone(), }; match serde_json::to_string(&lineup) { Ok(mut content) => { @@ -156,14 +157,13 @@ where Some(chans) => { let mapped = chans.map(move |(item, has_next)| { let lineup = Lineup { - guide_number: item.epg_channel_id.unwrap_or(item.name).clone(), + guide_number: item.epg_channel_id.clone().unwrap_or(item.name.clone()), guide_name: item.title.clone(), - url: (if item.t_stream_url.is_empty() { - &item.url + url: if item.t_stream_url.is_empty() { + item.url.clone() } else { - &item.t_stream_url - }) - .clone(), + item.t_stream_url.clone() + } }; match serde_json::to_string(&lineup) { Ok(mut content) => { @@ -230,7 +230,7 @@ async fn device_xml( .header(axum::http::header::CONTENT_TYPE, "application/xml") .body(axum::body::Body::from(device.as_xml()))) } else { - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } } @@ -240,7 +240,7 @@ async fn device_json( if let Some(device) = create_device(&app_state) { axum::Json(device).into_response() } else { - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } } @@ -250,7 +250,7 @@ async fn discover_json( if let Some(device) = create_device(&app_state) { axum::Json(device).into_response() } else { - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } } @@ -277,8 +277,7 @@ async fn lineup_status( cfg.get_target_for_username(&app_state.device.t_username) { if target.has_output(TargetType::M3u) { - if let Some((_guard, iter)) = - m3u_repository::iter_raw_m3u_playlist(&cfg, &target).await + if let Some((_guard, iter)) = m3u_repository::iter_raw_m3u_target_playlist(&cfg, &target, None).await { iter.count() } else { @@ -286,10 +285,8 @@ async fn lineup_status( } } else if target.has_output(TargetType::Xtream) { let credentials = Arc::new(user); - let live = - XtreamPlaylistIterator::new(XtreamCluster::Live, &cfg, &target, None, &credentials).await.map_or(0, std::iter::Iterator::count); - let vod = - XtreamPlaylistIterator::new(XtreamCluster::Video, &cfg, &target, None, &credentials).await.map_or(0, std::iter::Iterator::count); + let live = XtreamPlaylistIterator::new(XtreamCluster::Live, &cfg, &target, None, &credentials).await.map_or(0, std::iter::Iterator::count); + let vod = XtreamPlaylistIterator::new(XtreamCluster::Video, &cfg, &target, None, &credentials).await.map_or(0, std::iter::Iterator::count); live + vod } else { 0 diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 192074150..d39c66d9c 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -1,6 +1,6 @@ use crate::api::api_utils::{create_session_fingerprint, try_unwrap_body}; use crate::api::api_utils::{ - force_provider_stream_response, get_stream_alternative_url, is_seek_request, + force_provider_stream_response, get_stream_alternative_url, is_seek_request, local_stream_response, }; use crate::api::api_utils::{get_headers_from_request, try_option_bad_request, HeaderFilter}; use crate::api::model::AppState; @@ -21,8 +21,9 @@ use axum::response::IntoResponse; use log::{debug, error}; use serde::Deserialize; use shared::model::{PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, XtreamCluster}; -use shared::utils::{intern, is_hls_url, replace_url_extension, sanitize_sensitive_info, CUSTOM_VIDEO_PREFIX, HLS_EXT}; +use shared::utils::{is_hls_url, replace_url_extension, sanitize_sensitive_info, Internable, CUSTOM_VIDEO_PREFIX, HLS_EXT}; use std::sync::Arc; +use crate::utils::request::is_file_url; const PLAYLIST_TEMPLATE: &str = r"#EXTM3U #EXT-X-VERSION:3 @@ -102,7 +103,7 @@ pub(in crate::api) async fn handle_hls_stream_request( provider_cfg.name, sanitize_sensitive_info(&stream_url) ); - let user_session_token = create_session_fingerprint(&fingerprint.key, &user.username, virtual_id); + let user_session_token = create_session_fingerprint(fingerprint, &user.username, virtual_id); let session_token = app_state.active_users.create_user_session( user, &user_session_token, @@ -187,8 +188,9 @@ async fn resolve_stream_channel( app_state: &Arc, target: &Arc, virtual_id: u32, - hls_url: &str, + hls_url: &Arc, ) -> StreamChannel { + let unknown = "Unknown".intern(); let mut channel = match get_stream_channel(app_state, target, virtual_id).await { Some(channel) => channel, None => StreamChannel { @@ -197,9 +199,9 @@ async fn resolve_stream_channel( provider_id: 0, item_type: PlaylistItemType::LiveHls, cluster: XtreamCluster::Live, - group: intern("Unknown"), - title: "Unknown".to_string(), - url: hls_url.to_string(), + group: unknown.clone(), + title: unknown, + url: hls_url.clone(), shared: false, }, }; @@ -240,7 +242,7 @@ async fn hls_api_stream( ); debug_if_enabled!("ID chain for hls endpoint: request_stream_id={} -> virtual_id={virtual_id}", params.stream_id); - let user_session_token = create_session_fingerprint(&fingerprint.key, &user.username, virtual_id); + let user_session_token = create_session_fingerprint(&fingerprint, &user.username, virtual_id); let mut user_session = app_state .active_users .get_and_update_user_session(&user.username, &user_session_token).await; @@ -272,8 +274,8 @@ async fn hls_api_stream( Some((Some(session_token), hls_url)) if session.token.eq(&session_token) => hls_url, _ => return axum::http::StatusCode::BAD_REQUEST.into_response(), }; - - session.stream_url.clone_from(&hls_url); + let hls_url = hls_url.intern(); + session.stream_url = hls_url.clone(); if session.virtual_id == virtual_id { let stream_channel = resolve_stream_channel(&app_state, &target, virtual_id, &hls_url).await; if is_seek_request(stream_channel.cluster, &req_headers).await { @@ -314,8 +316,26 @@ async fn hls_api_stream( &req_headers, connection_permission, ) - .await - .into_response(); + .await + .into_response(); + } + + if is_file_url(&session.stream_url) { + let stream_channel = + resolve_stream_channel(&app_state, &target, virtual_id, &hls_url).await; + return local_stream_response( + &fingerprint, + &app_state, + stream_channel, + &req_headers, + &input, + &target, + &user, + connection_permission, + false, + ) + .await + .into_response(); } let stream_channel = resolve_stream_channel(&app_state, &target, virtual_id, &hls_url).await; diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index 48c643a26..6045d5b69 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -142,6 +142,7 @@ async fn m3u_api_stream( &target, &user, connection_permission, + true, ).await.into_response(); } @@ -150,7 +151,7 @@ async fn m3u_api_stream( debug_if_enabled!( "ID chain for m3u endpoint: request_stream_id={} -> action_stream_id={action_stream_id} -> req_virtual_id={req_virtual_id} -> virtual_id={virtual_id}", stream_req.stream_id); - let session_key = create_session_fingerprint(&fingerprint.key, &user.username, virtual_id); + let session_key = create_session_fingerprint(fingerprint, &user.username, virtual_id); let user_session = app_state .active_users .get_and_update_user_session(&user.username, &session_key).await; @@ -189,9 +190,9 @@ async fn m3u_api_stream( .await .into_response(); } - session.stream_url.as_str() + session.stream_url.clone() } else { - pli.url.as_str() + pli.url.clone() }; let connection_permission = user.connection_permission(app_state).await; @@ -252,7 +253,7 @@ async fn m3u_api_stream( app_state, &session_key, pli.to_stream_channel(target.id), - session_url, + &session_url, req_headers, &input, &target, diff --git a/backend/src/api/endpoints/user_api.rs b/backend/src/api/endpoints/user_api.rs index 48fab2dc3..c2263c469 100644 --- a/backend/src/api/endpoints/user_api.rs +++ b/backend/src/api/endpoints/user_api.rs @@ -29,9 +29,9 @@ fn get_categories_from_xtream(categories: Option>) - async fn get_categories_from_m3u_playlist(target: &ConfigTarget, config: &AppConfig) -> Vec> { let mut groups = Vec::new(); - if let Some((_guard, iter)) = m3u_repository::iter_raw_m3u_playlist(config, target).await { + if let Some((_guard, iter)) = m3u_repository::iter_raw_m3u_target_playlist(config, target, None).await { let mut unique_groups = HashSet::new(); - for (item, _has_next) in iter { + for item in iter { if !unique_groups.contains(&item.group) { unique_groups.insert(item.group.clone()); groups.push(item.group.clone()); diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index e06f103aa..269a07ba9 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -1,4 +1,4 @@ -use crate::api::api_utils::{json_or_bin_response, try_unwrap_body}; +use crate::api::api_utils::{json_or_bin_response, try_unwrap_body, internal_server_error}; use crate::api::endpoints::download_api; use crate::api::endpoints::user_api::user_api_register; use crate::api::endpoints::v1_api_playlist::v1_api_playlist_register; @@ -9,7 +9,7 @@ use crate::utils::ip_checker::get_ips; use crate::{VERSION}; use axum::response::IntoResponse; use shared::model::{default_geoip_url, InputFetchMethod, IpCheckDto, StatusCheck}; -use shared::utils::{concat_path_leading_slash}; +use shared::utils::{concat_path_leading_slash, Internable}; use std::collections::{BTreeMap, HashMap}; use std::io::{Cursor}; use std::sync::Arc; @@ -83,7 +83,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State { error!("Failed to process geoip db: {err}"); - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() }, - _ => { - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() - } + _ => internal_server_error!() } } Err(err) => { diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 9622d0e2c..2ea7e3faa 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -7,7 +7,7 @@ use shared::model::{ApiProxyConfigDto, ApiProxyServerInfoDto, ConfigDto, Sources use std::sync::Arc; use log::{error}; use shared::error::TuliproxError; -use crate::api::api_utils::try_unwrap_body; +use crate::api::api_utils::{try_unwrap_body, internal_server_error}; use crate::{utils}; use crate::utils::{prepare_sources_batch, prepare_users}; use crate::utils::request::download_text_content; @@ -124,17 +124,17 @@ async fn config( Ok(mut app_config) => { if let Err(err) = prepare_sources_batch(&mut app_config.sources, false).await { error!("Failed to prepare sources batch: {err}"); - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } else if let Err(err) = prepare_users(&mut app_config, &app_state.app_config).await { error!("Failed to prepare users: {err}"); - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } else { axum::response::Json(app_config).into_response() } } Err(err) => { error!("Failed to read config files: {err}"); - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } } } @@ -158,7 +158,7 @@ async fn config_batch_content( } Err(err) => { error!("Failed to read batch file: {err}"); - axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + internal_server_error!() } }; } diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index 1b3b4d528..66e1e6100 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -1,23 +1,24 @@ -use std::sync::Arc; -use axum::response::IntoResponse; -use axum::Router; -use log::error; -use serde_json::json; -use shared::model::{InputType, PlaylistEpgRequest, PlaylistRequest, WebplayerUrlRequest}; -use shared::utils::sanitize_sensitive_info; -use crate::api::endpoints::api_playlist_utils::{get_playlist, get_playlist_for_target}; +use crate::api::api_utils::{create_api_proxy_user, json_or_bin_response}; +use crate::api::endpoints::api_playlist_utils::{get_playlist_for_custom_provider, get_playlist_for_input, get_playlist_for_target}; +use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader; use crate::api::model::AppState; use crate::auth::create_access_token; use crate::model::{parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, ConfigInput, ConfigInputOptions}; use crate::processing::processor::playlist; +use axum::response::IntoResponse; +use axum::{Router}; +use log::{error, info}; +use serde_json::json; +use shared::model::{InputType, PlaylistEpgRequest, PlaylistRequest, ProxyType, TargetType, WebplayerUrlRequest, XtreamCluster}; +use shared::utils::{sanitize_sensitive_info, Internable}; +use std::sync::Arc; use url::Url; -use crate::api::api_utils::{json_or_bin_response}; -use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader; +use crate::api::endpoints::xtream_api::xtream_get_stream_info_response; fn create_config_input_for_m3u(url: &str) -> ConfigInput { ConfigInput { id: 0, - name: String::from("m3u_req"), + name: "m3u_req".intern(), input_type: InputType::M3u, url: String::from(url), enabled: true, @@ -35,7 +36,7 @@ fn create_config_input_for_m3u(url: &str) -> ConfigInput { fn create_config_input_for_xtream(username: &str, password: &str, host: &str) -> ConfigInput { ConfigInput { id: 0, - name: String::from("xc_req"), + name: "xc_req".intern(), input_type: InputType::Xtream, url: String::from(host), username: Some(String::from(username)), @@ -66,10 +67,10 @@ async fn playlist_update( let playlist_state = Arc::clone(&app_state.playlists); let valid_targets = Arc::new(valid_targets); tokio::spawn({ - async move { - playlist::exec_processing(&http_client, app_config, valid_targets, Some(event_manager), - Some(playlist_state), Some(app_state.update_guard.clone())).await; - } + async move { + playlist::exec_processing(&http_client, app_config, valid_targets, Some(event_manager), + Some(playlist_state), Some(app_state.update_guard.clone())).await; + } }); axum::http::StatusCode::ACCEPTED.into_response() } @@ -81,24 +82,25 @@ async fn playlist_update( } async fn playlist_content( - ExtractAcceptHeader(accept): ExtractAcceptHeader, - axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(playlist_req): axum::extract::Json, + accept: Option, + app_state: &Arc, + playlist_req: &PlaylistRequest, + cluster: XtreamCluster, ) -> impl IntoResponse + Send { let _config = app_state.app_config.config.load(); let client = app_state.http_client.load(); match playlist_req { PlaylistRequest::Target(target_id) => { - get_playlist_for_target(app_state.app_config.get_target_by_id(target_id).as_deref(), &app_state.app_config, accept.as_deref()).await.into_response() + get_playlist_for_target(app_state.app_config.get_target_by_id(*target_id).as_deref(), &app_state.app_config, cluster, accept.as_deref()).await.into_response() } PlaylistRequest::Input(input_id) => { - get_playlist(client.as_ref(), app_state.app_config.get_input_by_id(input_id).as_ref(), &app_state.app_config, accept.as_deref()).await.into_response() + get_playlist_for_input(app_state.app_config.get_input_by_id(*input_id).as_ref(), &app_state.app_config, cluster, accept.as_deref()).await.into_response() } PlaylistRequest::CustomXtream(xtream) => { match Url::parse(&xtream.url) { Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url)); - get_playlist(client.as_ref(), Some(&input), &app_state.app_config, accept.as_deref()).await.into_response() + get_playlist_for_custom_provider(client.as_ref(), Some(&input), &app_state.app_config, cluster, accept.as_deref()).await.into_response() } _ => { (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response() @@ -109,7 +111,7 @@ async fn playlist_content( match Url::parse(&m3u.url) { Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { let input = Arc::new(create_config_input_for_m3u(&m3u.url)); - get_playlist(client.as_ref(), Some(&input), &app_state.app_config, accept.as_deref()).await.into_response() + get_playlist_for_custom_provider(client.as_ref(), Some(&input), &app_state.app_config, cluster, accept.as_deref()).await.into_response() } _ => { (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response() @@ -119,6 +121,64 @@ async fn playlist_content( } } +macro_rules! create_player_api_for_cluster { + ($fn_name:ident, $cluster:expr) => { + async fn $fn_name( + ExtractAcceptHeader(accept): ExtractAcceptHeader, + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(playlist_req): axum::extract::Json, + ) -> impl IntoResponse + Send { + playlist_content( + accept.clone(), + &app_state, + &playlist_req, + $cluster + ) + .await + .into_response() + } + }; +} + +create_player_api_for_cluster!(playlist_content_live, XtreamCluster::Live); +create_player_api_for_cluster!(playlist_content_vod, XtreamCluster::Video); +create_player_api_for_cluster!(playlist_content_series, XtreamCluster::Series); + +async fn playlist_series_info( + axum::extract::Path((virtual_id, _provider_id)): axum::extract::Path<( + String, + String, + )>, + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(playlist_req): axum::extract::Json, +) -> impl IntoResponse + Send { + match playlist_req { + PlaylistRequest::Target(target_id) => { + if let Some(target) = app_state.app_config.get_target_by_id(target_id) { + if target.has_output(TargetType::Xtream) { + let mut user = create_api_proxy_user(&app_state); + user.proxy = ProxyType::Redirect; + return xtream_get_stream_info_response(&app_state, &user, &target, &virtual_id, XtreamCluster::Series).await.into_response(); + } + } + } + PlaylistRequest::Input(input_id) => { + if let Some(input) = app_state.app_config.get_input_by_id(input_id) { + if matches!(input.input_type, InputType::Xtream | InputType::XtreamBatch) { + // TODO: Implement series info retrieval for input-based requests + info!("TODO: Implement series info retrieval for input-based requests"); + } + } + } + PlaylistRequest::CustomXtream(_xtream) => { + // TODO: Implement series info retrieval for custom Xtream requests + info!("TODO: Implement series info retrieval for custom Xtream requests"); + }, + PlaylistRequest::CustomM3u(_) => {} + } + axum::http::StatusCode::NO_CONTENT.into_response() +} + async fn playlist_webplayer( axum::extract::State(app_state): axum::extract::State>, axum::extract::Json(playlist_item): axum::extract::Json, @@ -140,7 +200,7 @@ async fn playlist_epg( PlaylistEpgRequest::Target(target_id) => { if let Some(target) = app_state.app_config.get_target_by_id(target_id) { let config = &app_state.app_config.config.load(); - if let Some(epg_path) = crate::api::endpoints::xmltv_api::get_epg_path_for_target(config, &target) { + if let Some(epg_path) = crate::api::endpoints::xmltv_api::get_epg_path_for_target(config, &target) { if let Ok(epg) = parse_xmltv_for_web_ui_from_file(&epg_path).await { return json_or_bin_response(accept.as_deref(), &epg).into_response(); } @@ -149,14 +209,14 @@ async fn playlist_epg( } PlaylistEpgRequest::Input(_input_id) => { // TODO: This is currently not supported, because we could have multiple epg sources for one input - // if let Some(target) = app_state.app_config.get_input_by_id(input_id) { - // let config = &app_state.app_config.config.load(); - // if let Some(epg_path) = crate::api::endpoints::xmltv_api::get_epg_path_for_input(config, &target) { - // if let Ok(epg) = parse_xmltv_for_web_ui(&epg_path) { - // return json_or_bin_response(accept.as_ref(), &epg).into_response(); - // } - // } - // } + // if let Some(target) = app_state.app_config.get_input_by_id(input_id) { + // let config = &app_state.app_config.config.load(); + // if let Some(epg_path) = crate::api::endpoints::xmltv_api::get_epg_path_for_input(config, &target) { + // if let Ok(epg) = parse_xmltv_for_web_ui(&epg_path) { + // return json_or_bin_response(accept.as_ref(), &epg).into_response(); + // } + // } + // } } PlaylistEpgRequest::Custom(url) => { if let Ok(epg) = parse_xmltv_for_web_ui_from_url(&app_state, &url).await { @@ -172,5 +232,8 @@ pub fn v1_api_playlist_register(router: Router>) -> axum::Router, - filter: Option, + filter: Option>, ) -> axum::response::Response { if let Ok(exists) = tokio::fs::try_exists(epg_path).await { if exists { @@ -188,7 +188,7 @@ async fn serve_epg_with_rewrites( rewrite_urls: bool, secret: &[u8; 16], base_url: &str, - filter: Option, + filter: Option>, ) -> axum::response::Response { match tokio::fs::try_exists(epg_path).await { Ok(exists) => { @@ -256,14 +256,14 @@ async fn serve_epg_with_rewrites( .filter_map(Result::ok) .find(|a| a.key.as_ref() == b"id") .and_then(|a| a.unescape_value().ok()) - .is_some_and(|v| !flt.eq(v.as_ref())) + .is_some_and(|v| flt.as_ref() != v.as_ref()) } b"programme" => { e.attributes() .filter_map(Result::ok) .find(|a| a.key.as_ref() == b"channel") .and_then(|a| a.unescape_value().ok()) - .is_some_and(|v| !flt.eq(v.as_ref())) + .is_some_and(|v| flt.as_ref() != v.as_ref()) } _ => false, }; @@ -409,7 +409,7 @@ async fn serve_epg_with_rewrites( .header(axum::http::header::CONTENT_ENCODING, "gzip") // Set Content-Encoding header .body(axum::body::Body::from_stream(body_stream))) } - Err(_) => axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Err(_) => internal_server_error!(), } } @@ -446,7 +446,6 @@ async fn xmltv_api( serve_epg(&app_state, &epg_path, &user, &target, None).await } -#[axum::debug_handler] async fn epg_api_resource( req_headers: axum::http::HeaderMap, axum::extract::Query(api_req): axum::extract::Query, diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index e7308cd72..877d58bdb 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1,13 +1,11 @@ // https://github.com/tellytv/go.xtream-codes/blob/master/structs.go // Xtream api -> https://9tzx6f0ozj.apidog.io/ use crate::api::api_utils; -use crate::api::api_utils::{create_session_fingerprint, local_stream_response, try_unwrap_body}; -use crate::api::api_utils::{ - force_provider_stream_response, get_user_target, get_user_target_by_credentials, - is_seek_request, redirect_response, resource_response, separate_number_and_remainder, - stream_response, RedirectParams, -}; -use crate::api::api_utils::{redirect, try_option_bad_request, try_result_bad_request, try_result_not_found}; +use crate::api::api_utils::{create_api_proxy_user, create_session_fingerprint, force_provider_stream_response, + get_user_target, get_user_target_by_credentials, internal_server_error, + is_seek_request, local_stream_response, redirect, redirect_response, + resource_response, separate_number_and_remainder, stream_response, try_option_bad_request, try_result_bad_request, try_result_not_found, + try_unwrap_body, RedirectParams}; use crate::api::endpoints::hls_api::handle_hls_stream_request; use crate::api::endpoints::xmltv_api::{get_empty_epg_response, get_epg_path_for_target, serve_epg}; use crate::api::model::AppState; @@ -33,13 +31,15 @@ use futures::Stream; use log::{debug, error, warn}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use shared::error::{info_err_res, info_err, TuliproxError}; -use shared::model::{create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType, TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{deserialize_as_string, extract_extension_from_url, generate_playlist_uuid, sanitize_sensitive_info, trim_slash, HLS_EXT}; +use shared::concat_string; +use shared::error::{info_err, info_err_res, TuliproxError}; +use shared::model::{create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType, + TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem}; +use shared::utils::{deserialize_as_string, extract_extension_from_url, generate_playlist_uuid, + sanitize_sensitive_info, trim_slash, HLS_EXT}; use std::fmt::{Display, Formatter}; use std::str::FromStr; use std::sync::Arc; -use shared::concat_string; #[derive(Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq)] pub enum ApiStreamContext { @@ -153,8 +153,8 @@ pub(in crate::api) fn get_xtream_player_api_stream_url( input: &ConfigInput, context: ApiStreamContext, action_path: &str, - fallback_url: &str, -) -> Option { + fallback_url: &Arc, +) -> Option> { if let Some(input_user_info) = input.get_user_info() { let ctx = match context { ApiStreamContext::LiveAlt | ApiStreamContext::Live => { @@ -176,9 +176,9 @@ pub(in crate::api) fn get_xtream_player_api_stream_url( trim_slash(action_path), ]; parts.retain(|s| !s.is_empty()); - Some(parts.join("/")) + Some(parts.join("/").into()) } else if !fallback_url.is_empty() { - Some(String::from(fallback_url)) + Some(fallback_url.clone()) } else { None } @@ -266,6 +266,7 @@ async fn xtream_player_api_stream( &target, &user, connection_permission, + true, ).await.into_response(); } @@ -278,7 +279,7 @@ async fn xtream_player_api_stream( debug_if_enabled!( "ID chain for xtream endpoint: request_stream_id={} -> action_stream_id={action_stream_id} -> req_virtual_id={req_virtual_id} -> virtual_id={virtual_id}", stream_req.stream_id); - let session_key = create_session_fingerprint(&fingerprint.key, &user.username, virtual_id); + let session_key = create_session_fingerprint(fingerprint, &user.username, virtual_id); let user_session = app_state .active_users .get_and_update_user_session(&user.username, &session_key).await; @@ -321,9 +322,9 @@ async fn xtream_player_api_stream( .into_response(); } - session.stream_url.as_str() + session.stream_url.clone() } else { - pli.url.as_str() + pli.url.clone() }; let connection_permission = user.connection_permission(app_state).await; @@ -356,7 +357,7 @@ async fn xtream_player_api_stream( let (query_path, extension) = get_query_path(stream_req.action_path, stream_ext.as_ref(), &pli, app_state); let stream_url = try_option_bad_request!( - get_xtream_player_api_stream_url(&input, stream_req.context, &query_path, session_url), + get_xtream_player_api_stream_url(&input, stream_req.context, &query_path, &session_url), true, format!( "Can't find stream url for target {target_name}, context {}, stream_id {virtual_id}", @@ -403,8 +404,13 @@ async fn xtream_player_api_stream( } fn get_query_path(action_path: &str, stream_ext: Option<&String>, pli: &XtreamPlaylistItem, app_state: &Arc) -> (String, String) { - - let discard_extension = pli.item_type.is_live() && app_state.app_config.sources.load().get_input_by_name(&pli.input_name).as_ref().and_then(|i| i.options.as_ref()).is_some_and(|o| o.xtream_live_stream_without_extension); + let discard_extension = if pli.item_type.is_live() { + app_state.app_config.sources.load().get_input_by_name(&pli.input_name) + .as_ref().and_then(|i| i.options.as_ref()) + .is_some_and(|o| o.xtream_live_stream_without_extension) + } else { + false + }; let extracted_ext; let extension: &str = if discard_extension { @@ -456,9 +462,7 @@ async fn xtream_player_api_stream_with_token( ); let virtual_id = pli.virtual_id; let input = try_option_bad_request!( - app_state - .app_config - .get_input_by_name(&pli.input_name), + app_state.app_config.get_input_by_name(&pli.input_name), true, format!( "Can't find input {} for target {target_name}, context {}, stream_id {}", @@ -466,42 +470,23 @@ async fn xtream_player_api_stream_with_token( ) ); - let config = app_state.app_config.config.load(); - - let server = config - .web_ui - .as_ref() - .and_then(|web_ui| web_ui.player_server.as_ref()) - .map_or("default", |server_name| server_name.as_str()); - - let user = ProxyUserCredentials { - username: "api_user".to_string(), - password: "api_user".to_string(), - token: None, - proxy: ProxyType::Reverse(None), - server: Some(server.to_string()), - epg_timeshift: None, - created_at: None, - exp_date: None, - max_connections: 0, - status: None, - ui_enabled: false, - comment: None, - }; + let user = create_api_proxy_user(app_state); if pli.item_type.is_local() { - return local_stream_response(fingerprint, - app_state, - pli.to_stream_channel(target.id), - req_headers, - &input, - &target, - &user, - UserConnectionPermission::Allowed, + return local_stream_response( + fingerprint, + app_state, + pli.to_stream_channel(target.id), + req_headers, + &input, + &target, + &user, + UserConnectionPermission::Allowed, + true, ).await.into_response(); } - let session_key = create_session_fingerprint(&fingerprint.key, "webui", virtual_id); + let session_key = create_session_fingerprint(fingerprint, "webui", virtual_id); let is_hls_request = pli.item_type == PlaylistItemType::LiveHls || stream_ext.as_deref() == Some(HLS_EXT); @@ -532,7 +517,7 @@ async fn xtream_player_api_stream_with_token( &input, stream_req.context, &query_path, - pli.url.as_str() + &pli.url ), true, format!( @@ -802,7 +787,7 @@ async fn xtream_player_api_timeshift_query_stream( .into_response() } -fn empty_json_response() -> axum::http::Result { +fn empty_json_response_as_object() -> axum::http::Result { axum::response::Response::builder() .status(axum::http::StatusCode::OK) .header( @@ -812,7 +797,18 @@ fn empty_json_response() -> axum::http::Result { .body(axum::body::Body::from("{}".as_bytes())) } -async fn xtream_get_stream_info_response( +fn empty_json_response_as_array() -> axum::http::Result { + axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header( + axum::http::header::CONTENT_TYPE, + mime::APPLICATION_JSON.to_string(), + ) + .body(axum::body::Body::from("[]".as_bytes())) +} + + +pub async fn xtream_get_stream_info_response( app_state: &Arc, user: &ProxyUserCredentials, target: &Arc, @@ -821,7 +817,7 @@ async fn xtream_get_stream_info_response( ) -> impl IntoResponse + Send { let virtual_id: u32 = match FromStr::from_str(stream_id) { Ok(id) => id, - Err(_) => return axum::http::StatusCode::BAD_REQUEST.into_response(), + Err(_) => return try_unwrap_body!(empty_json_response_as_array()), }; if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id( @@ -832,7 +828,7 @@ async fn xtream_get_stream_info_response( ).await { if pli.item_type.is_local() { let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected: xtream output required for target {}", target.name)) else { - return try_unwrap_body!(empty_json_response()); + return try_unwrap_body!(empty_json_response_as_array()); }; let server_info = app_state.app_config.get_user_server_info(user); @@ -883,17 +879,11 @@ async fn xtream_get_stream_info_response( .body(axum::body::Body::from(content))) } XtreamCluster::Live | XtreamCluster::Series => { - try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header( - axum::http::header::CONTENT_TYPE, - mime::APPLICATION_JSON.to_string() - ) - .body(axum::body::Body::from("{}".as_bytes()))) + try_unwrap_body!(empty_json_response_as_array()) } }; } - try_unwrap_body!(empty_json_response()) + try_unwrap_body!(empty_json_response_as_object()) } async fn xtream_get_short_epg( @@ -907,7 +897,7 @@ async fn xtream_get_short_epg( if target.has_output(TargetType::Xtream) { let virtual_id: u32 = match FromStr::from_str(stream_id.trim()) { Ok(id) => id, - Err(_) => return axum::http::StatusCode::BAD_REQUEST.into_response(), + Err(_) => return get_empty_epg_response().into_response(), }; if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id( @@ -920,7 +910,7 @@ async fn xtream_get_short_epg( if let Some(epg_path) = get_epg_path_for_target(config, target) { if let Ok(exists) = tokio::fs::try_exists(&epg_path).await { if exists { - return serve_epg(app_state, &epg_path, user, target, pli.epg_channel_id.clone()).await; + return serve_epg(app_state, &epg_path, user, target, pli.epg_channel_id).await; } } } @@ -954,7 +944,7 @@ async fn xtream_get_short_epg( &input_source, None, None, - false + false, ) .await { @@ -1015,7 +1005,7 @@ async fn xtream_player_api_handle_content_action( categories.retain(|c| fltr.contains(&c.category_id)); } return Some(axum::Json(categories).into_response()); - }, + } Err(err) => error!("Failed to parse json file {}: {err}", file_path.display()), } } @@ -1068,8 +1058,9 @@ async fn xtream_get_catchup_response( let config = &app_state.app_config.config.load(); let target_path = try_option_bad_request!(get_target_storage_path(config, target.name.as_str())); - let (mut target_id_mapping, file_lock) = - get_target_id_mapping(&app_state.app_config, &target_path).await; + let Ok((mut target_id_mapping, file_lock)) = get_target_id_mapping(&app_state.app_config, &target_path, target.use_memory_cache).await else { + return internal_server_error!() + }; let mut in_memory_updates = Vec::new(); for epg_list_item in epg_listings.iter_mut().filter_map(Value::as_object_mut) { // TODO epg_id @@ -1292,7 +1283,7 @@ async fn xtream_player_api( ) .await ), - _ => Some(info_err_res!("Can't find content: {action} for target: {}", &target.name)), + _ => Some(info_err_res!("Unknown api call: {action} for target: {}", &target.name)), }; match result { diff --git a/backend/src/api/hdhomerun_proprietary.rs b/backend/src/api/hdhomerun_proprietary.rs index f6c39fa4b..dea641365 100644 --- a/backend/src/api/hdhomerun_proprietary.rs +++ b/backend/src/api/hdhomerun_proprietary.rs @@ -10,6 +10,7 @@ use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, UdpSocket}; use tokio_util::sync::CancellationToken; +use shared::utils::Internable; const HDHR_PROPRIETARY_PORT: u16 = 65001; @@ -370,7 +371,7 @@ async fn process_getset_request(request: &[u8], app_state: &Arc) -> Ve let vchannel = if let Some(stream_info) = active_streams.get(tuner_index) { stream_info.channel.title.clone() } else { - "none".to_string() + "none".intern() }; write_tlv_str( &mut response_payload, diff --git a/backend/src/api/main_api.rs b/backend/src/api/main_api.rs index 522abf46a..573b9547a 100644 --- a/backend/src/api/main_api.rs +++ b/backend/src/api/main_api.rs @@ -12,7 +12,7 @@ use crate::api::endpoints::xtream_api::xtream_api_register; use crate::api::hdhomerun_proprietary::spawn_proprietary_tasks; use crate::api::hdhomerun_ssdp::spawn_ssdp_discover_task; use crate::api::model::{create_cache, create_http_client, ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, DownloadQueue, EventManager, HdHomerunAppState, PlaylistStorageState, SharedStreamManager, UpdateGuard}; -use crate::api::scheduler::exec_scheduler; +use crate::api::scheduler::{exec_interner_prune, exec_scheduler}; use crate::api::serve::serve; use crate::model::{AppConfig, Config, Healthcheck, ProcessTargets, RateLimitConfig}; use crate::processing::processor::playlist; @@ -296,6 +296,8 @@ pub async fn start_server( exec_file_lock_prune(&app_state); + exec_interner_prune(&app_state); + exec_config_watch(&app_state, &cancel_token_file_watch); let web_auth_enabled = is_web_auth_enabled(&cfg, web_ui_enabled); @@ -310,10 +312,7 @@ pub async fn start_server( .and_then(|c| c.path.as_ref()) .cloned() .unwrap_or_default(); - infos.push(format!( - "Server running: http://{}:{}", - &cfg.api.host, &cfg.api.port - )); + infos.push(format!("Server running: http://{}:{}", &cfg.api.host, &cfg.api.port)); for info in &infos { info!("{info}"); } diff --git a/backend/src/api/model/active_provider_manager.rs b/backend/src/api/model/active_provider_manager.rs index 64a485899..0c63a510f 100644 --- a/backend/src/api/model/active_provider_manager.rs +++ b/backend/src/api/model/active_provider_manager.rs @@ -85,7 +85,7 @@ impl ActiveProviderManager { } pub async fn reconcile_connections(&self) { - let mut counts = HashMap::::new(); + let mut counts = HashMap::, usize>::new(); { let connections = self.connections.read().await; @@ -111,7 +111,7 @@ impl ActiveProviderManager { async fn acquire_connection_inner( &self, - provider_or_input_name: &str, + provider_or_input_name: &Arc, addr: &SocketAddr, force: bool, allow_grace_override: Option, @@ -152,35 +152,35 @@ impl ActiveProviderManager { None } - pub async fn force_exact_acquire_connection(&self, provider_name: &str, addr: &SocketAddr) -> Option { + pub async fn force_exact_acquire_connection(&self, provider_name: &Arc, addr: &SocketAddr) -> Option { self.acquire_connection_inner(provider_name, addr, true, None).await } // Returns the next available provider connection - pub async fn acquire_connection(&self, input_name: &str, addr: &SocketAddr) -> Option { + pub async fn acquire_connection(&self, input_name: &Arc, addr: &SocketAddr) -> Option { self.acquire_connection_inner(input_name, addr, false, None).await } /// Acquire a provider connection while optionally disabling provider grace allocations. pub async fn acquire_connection_with_grace_override( &self, - input_name: &str, + input_name: &Arc, addr: &SocketAddr, allow_grace: bool, ) -> Option { self.acquire_connection_inner(input_name, addr, false, Some(allow_grace)).await } - // This method is used for redirects to cycle through provider - pub async fn get_next_provider(&self, provider_name: &str) -> Option> { + // This method is used for redirects to cycle through the provider + pub async fn get_next_provider(&self, provider_name: &Arc) -> Option> { self.providers.get_next_provider(provider_name).await } - pub async fn active_connections(&self) -> Option> { + pub async fn active_connections(&self) -> Option, usize>> { self.providers.active_connections().await } - pub async fn is_over_limit(&self, provider_name: &str) -> bool { + pub async fn is_over_limit(&self, provider_name: &Arc) -> bool { self.providers.is_over_limit(provider_name).await } diff --git a/backend/src/api/model/active_user_manager.rs b/backend/src/api/model/active_user_manager.rs index 3088f1360..c04cc2e8a 100644 --- a/backend/src/api/model/active_user_manager.rs +++ b/backend/src/api/model/active_user_manager.rs @@ -7,7 +7,8 @@ use arc_swap::ArcSwapOption; use jsonwebtoken::get_current_timestamp; use log::{debug, info}; use shared::model::{ActiveUserConnectionChange, StreamChannel, StreamInfo, UserConnectionPermission, VirtualId}; -use shared::utils::{current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, intern, sanitize_sensitive_info, strip_port}; +use shared::utils::{current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, + sanitize_sensitive_info, strip_port, Internable}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; @@ -31,8 +32,8 @@ fn get_grace_options(config: &Config) -> (u64, u64) { pub struct UserSession { pub token: String, pub virtual_id: u32, - pub provider: String, - pub stream_url: String, + pub provider: Arc, + pub stream_url: Arc, pub addr: SocketAddr, pub ts: u64, pub permission: UserConnectionPermission, @@ -250,8 +251,8 @@ impl ActiveUserManager { for stream in &mut connection_data.streams { if &stream.addr == addr { stream.provider = String::from("tuliprox"); - stream.channel.title = video_type.to_string(); - stream.channel.group = intern(""); + stream.channel.title = video_type.to_string().into(); + stream.channel.group = "".intern(); return Some(stream.clone()); } } @@ -351,8 +352,8 @@ impl ActiveUserManager { UserSession { token: session_token.to_string(), virtual_id, - provider: provider.to_string(), - stream_url: stream_url.to_string(), + provider: provider.intern(), + stream_url: stream_url.intern(), addr: *addr, ts: current_time_secs(), permission: connection_permission, @@ -379,11 +380,11 @@ impl ActiveUserManager { for session in &mut connection_data.sessions { if session.token == session_token { session.ts = current_time_secs(); - if session.stream_url != stream_url { - session.stream_url = stream_url.to_string(); + if &*session.stream_url != stream_url { + session.stream_url = stream_url.intern(); } - if session.provider != provider { - session.provider = provider.to_string(); + if &*session.provider != provider { + session.provider = provider.intern(); } session.permission = connection_permission; debug_if_enabled!("Using session for user {} with url: {}", user.username, sanitize_sensitive_info(stream_url)); diff --git a/backend/src/api/model/event_manager.rs b/backend/src/api/model/event_manager.rs index 32cc45881..1bfa992b4 100644 --- a/backend/src/api/model/event_manager.rs +++ b/backend/src/api/model/event_manager.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use log::{trace}; use shared::model::{ActiveUserConnectionChange, ConfigType, LibraryScanSummary, PlaylistUpdateState, SystemInfo}; @@ -6,7 +7,7 @@ use shared::model::{ActiveUserConnectionChange, ConfigType, LibraryScanSummary, pub enum EventMessage { ServerError(String), ActiveUser(ActiveUserConnectionChange), // user_count, connection count - ActiveProvider(String, usize), // provider name, connections + ActiveProvider(Arc, usize), // provider name, connections ConfigChange(ConfigType), PlaylistUpdate(PlaylistUpdateState), PlaylistUpdateProgress(String, String), @@ -41,8 +42,8 @@ impl EventManager { } } - pub fn send_provider_event(&self, provider: &str, connection_count: usize) { - if !self.send_event(EventMessage::ActiveProvider(String::from(provider), connection_count)) { + pub fn send_provider_event(&self, provider: &Arc, connection_count: usize) { + if !self.send_event(EventMessage::ActiveProvider(Arc::clone(provider), connection_count)) { trace!("Failed to send connection change: {provider}: {connection_count}"); } } diff --git a/backend/src/api/model/provider_config.rs b/backend/src/api/model/provider_config.rs index 47f8ba4aa..23c268315 100644 --- a/backend/src/api/model/provider_config.rs +++ b/backend/src/api/model/provider_config.rs @@ -12,7 +12,7 @@ use crate::api::model::ProviderAllocation; use crate::utils::debug_if_enabled; -pub type ProviderConnectionChangeCallback = Arc; +pub type ProviderConnectionChangeCallback = Arc, usize) + Send + Sync>; #[derive(Debug, Clone, Copy)] pub enum ProviderConfigAllocation { @@ -37,7 +37,7 @@ pub struct ProviderConfigConnection { /// `current_connections`: A `RwLock` to safely track the number of active connections. pub struct ProviderConfig { pub id: u16, - pub name: String, + pub name: Arc, pub url: String, pub username: Option, pub password: Option, diff --git a/backend/src/api/model/provider_lineup_manager.rs b/backend/src/api/model/provider_lineup_manager.rs index dbd82e70e..226322413 100644 --- a/backend/src/api/model/provider_lineup_manager.rs +++ b/backend/src/api/model/provider_lineup_manager.rs @@ -18,7 +18,7 @@ macro_rules! gen_provider_search { for lineup in providers { match lineup { ProviderLineup::Single(single) => { - if single.provider.$field == criteria { + if &single.provider.$field == criteria { return Some((lineup, &single.provider)); } } @@ -26,13 +26,13 @@ macro_rules! gen_provider_search { for group in &multi.providers { match group { ProviderPriorityGroup::SingleProviderGroup(single) => { - if single.$field == criteria { + if &single.$field == criteria { return Some((lineup, single)); } } ProviderPriorityGroup::MultiProviderGroup(_, configs) => { for config in configs { - if config.$field == criteria { + if &config.$field == criteria { return Some((lineup, config)); } } @@ -48,11 +48,11 @@ macro_rules! gen_provider_search { } fn get_or_create_provider_connection( - provider_connections: &DashMap>>, - provider_name: &str, + provider_connections: &DashMap, Arc>>, + provider_name: &Arc, ) -> Arc> { provider_connections - .entry(provider_name.to_string()) + .entry(provider_name.clone()) .or_insert_with(|| Arc::new(RwLock::new(ProviderConfigConnection::default()))) .clone() } @@ -81,7 +81,7 @@ impl ProviderAllocation { ProviderAllocation::GracePeriod(config) } - pub fn get_provider_name(&self) -> Option { + pub fn get_provider_name(&self) -> Option> { match self { ProviderAllocation::Exhausted => None, ProviderAllocation::Available(ref cfg) | @@ -200,8 +200,8 @@ impl SingleProviderLineup { } #[cfg(test)] - async fn release(&self, provider_name: &str) { - if self.provider.name == provider_name { + async fn release(&self, provider_name: &Arc) { + if &self.provider.name == provider_name { self.provider.release().await; } } @@ -235,21 +235,21 @@ impl fmt::Display for ProviderPriorityGroup { #[repr(align(64))] #[derive(Debug)] struct MultiProviderLineup { - name: String, + name: Arc, providers: Vec, } impl MultiProviderLineup { pub fn new( cfg_input: &ConfigInput, - provider_connections: &DashMap>>, + provider_connections: &DashMap, Arc>>, connection_change: &ProviderConnectionChangeCallback, ) -> Self { - let input_connection = get_or_create_provider_connection(provider_connections, cfg_input.name.as_str()); + let input_connection = get_or_create_provider_connection(provider_connections, &cfg_input.name); let mut inputs = vec![ProviderConfigWrapper::new(ProviderConfig::new(cfg_input, input_connection, Arc::clone(connection_change)))]; if let Some(aliases) = &cfg_input.aliases { for alias in aliases { - let alias_connection = get_or_create_provider_connection(provider_connections, alias.name.as_str()); + let alias_connection = get_or_create_provider_connection(provider_connections, &alias.name); inputs.push(ProviderConfigWrapper::new(ProviderConfig::new_alias( cfg_input, alias, @@ -464,18 +464,18 @@ impl MultiProviderLineup { #[cfg(test)] - async fn release(&self, provider_name: &str) { + async fn release(&self, provider_name: &Arc) { for g in &self.providers { match g { ProviderPriorityGroup::SingleProviderGroup(pc) => { - if pc.name == provider_name { + if &pc.name == provider_name { pc.release().await; break; } } ProviderPriorityGroup::MultiProviderGroup(_, group) => { for pc in group { - if pc.name == provider_name { + if &pc.name == provider_name { pc.release().await; return; } @@ -509,13 +509,13 @@ pub(in crate::api::model) struct ProviderLineupManager { grace_period_timeout_secs: AtomicU64, inputs: Arc>>>, providers: Arc>>, - provider_connections: DashMap>>, + provider_connections: DashMap, Arc>>, event_manager: Arc, } impl ProviderLineupManager { pub fn new(inputs: Vec>, grace_period_millis: u64, grace_period_timeout_secs: u64, event_manager: &Arc) -> Self { - let provider_connections: DashMap>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineups = inputs .iter() .map(|i| Self::create_lineup(i, &provider_connections, event_manager)) @@ -532,18 +532,18 @@ impl ProviderLineupManager { fn create_lineup( cfg_input: &ConfigInput, - provider_connections: &DashMap>>, + provider_connections: &DashMap, Arc>>, event_manager: &Arc, ) -> ProviderLineup { let event_manager = Arc::clone(event_manager); - let on_connection_change: ProviderConnectionChangeCallback = Arc::new(move |name: &str, connections: usize| { + let on_connection_change: ProviderConnectionChangeCallback = Arc::new(move |name: &Arc, connections: usize| { event_manager.send_provider_event(name, connections); }); if cfg_input.aliases.as_ref().is_some_and(|a| !a.is_empty()) { ProviderLineup::Multi(MultiProviderLineup::new(cfg_input, provider_connections, &on_connection_change)) } else { - let connection = get_or_create_provider_connection(provider_connections, cfg_input.name.as_str()); + let connection = get_or_create_provider_connection(provider_connections, &cfg_input.name); ProviderLineup::Single(SingleProviderLineup::new(cfg_input, connection, &on_connection_change)) } } @@ -628,7 +628,7 @@ impl ProviderLineupManager { self.providers.store(Arc::new(new_lineups)); } - pub async fn reconcile_connections(&self, mut counts: HashMap) { + pub async fn reconcile_connections(&self, mut counts: HashMap, usize>) { // 1. Synchronize known providers from actual counts. // This avoids a transient "zero state" by updating each provider in one step. for entry in &self.provider_connections { @@ -669,7 +669,7 @@ impl ProviderLineupManager { // nor referenced by any active stream allocations. // // deterministic check: Is the name still in any current lineup? - let current_names: std::collections::HashSet = { + let current_names: std::collections::HashSet> = { let mut names = std::collections::HashSet::new(); let lineups = self.providers.load(); for lineup in lineups.iter() { @@ -704,7 +704,7 @@ impl ProviderLineupManager { }); } - gen_provider_search!(get_provider_config_by_name, name, &str); + gen_provider_search!(get_provider_config_by_name, name, &Arc); fn log_allocation(allocation: &ProviderAllocation) { @@ -723,7 +723,7 @@ impl ProviderLineupManager { } } - pub async fn force_exact_acquire_connection(&self, provider_name: &str) -> ProviderAllocation { + pub async fn force_exact_acquire_connection(&self, provider_name: &Arc) -> ProviderAllocation { let providers = self.providers.load(); let allocation = match Self::get_provider_config_by_name(provider_name, &providers) { None => ProviderAllocation::Exhausted, // No Name matched, we don't have this provider @@ -734,7 +734,7 @@ impl ProviderLineupManager { } // Returns the next available provider connection - pub(crate) async fn acquire_connection(&self, input_name: &str) -> ProviderAllocation { + pub(crate) async fn acquire_connection(&self, input_name: &Arc) -> ProviderAllocation { self.acquire_connection_with_grace_override(input_name, true).await } @@ -744,7 +744,7 @@ impl ProviderLineupManager { /// even if a global grace period is configured. pub(crate) async fn acquire_connection_with_grace_override( &self, - input_name: &str, + input_name: &Arc, allow_grace: bool, ) -> ProviderAllocation { let providers = self.providers.load(); @@ -809,7 +809,7 @@ impl ProviderLineupManager { // This method is used for redirects to cycle through provider // - pub async fn get_next_provider(&self, input_name: &str) -> Option> { + pub async fn get_next_provider(&self, input_name: &Arc) -> Option> { let providers = self.providers.load(); match Self::get_provider_config_by_name(input_name, &providers) { None => None, @@ -825,11 +825,11 @@ impl ProviderLineupManager { } } - pub async fn active_connections(&self) -> Option> { - let mut result = HashMap::::new(); - let mut add_provider = |name: String, count: usize| { + pub async fn active_connections(&self) -> Option, usize>> { + let mut result = HashMap::, usize>::new(); + let mut add_provider = |name: &Arc, count: usize| { if count > 0 { - result.insert(name, count); + result.insert(name.clone(), count); } }; let providers = self.providers.load(); @@ -837,11 +837,11 @@ impl ProviderLineupManager { match lineup { ProviderLineup::Single(provider_lineup) => { let connections = provider_lineup.provider.get_current_connections().await; - add_provider(provider_lineup.provider.name.clone(), connections); + add_provider(&provider_lineup.provider.name, connections); } ProviderLineup::Multi(provider_lineup) => { let connections = provider_lineup.get_total_connections().await; - add_provider(provider_lineup.name.clone(), connections); + add_provider(&provider_lineup.name, connections); } } } @@ -868,7 +868,7 @@ impl ProviderLineupManager { count } - pub async fn is_over_limit(&self, provider_name: &str) -> bool { + pub async fn is_over_limit(&self, provider_name: &Arc) -> bool { let providers = self.providers.load(); if let Some((_, config)) = Self::get_provider_config_by_name(provider_name, &providers) { config.is_over_limit(self.grace_period_timeout_secs.load(Ordering::Relaxed)).await @@ -887,6 +887,8 @@ mod tests { use shared::model::{InputFetchMethod, InputType}; use std::sync::atomic::AtomicU16; use std::thread; + use shared::concat_string; + use shared::utils::Internable; macro_rules! should_available { ($lineup:expr, $provider_id:expr, $grace_period_timeout_secs: expr) => { @@ -921,10 +923,10 @@ mod tests { } // Helper function to create a ConfigInput instance - fn create_config_input(id: u16, name: &str, priority: i16, max_connections: u16) -> ConfigInput { + fn create_config_input(id: u16, name: &Arc, priority: i16, max_connections: u16) -> ConfigInput { ConfigInput { id, - name: name.to_string(), + name: Arc::clone(name), url: "http://example.com".to_string(), epg: Option::default(), username: None, @@ -950,7 +952,7 @@ mod tests { fn create_config_input_alias(id: u16, url: &str, priority: i16, max_connections: u16) -> ConfigInputAlias { ConfigInputAlias { id, - name: format!("alias_{id}"), + name: concat_string!("alias_", &id.to_string()).intern(), url: url.to_string(), username: Some("alias_user".to_string()), password: Some("alias_pass".to_string()), @@ -960,20 +962,20 @@ mod tests { } } - fn dummy_callback(_: &str, _: usize) {} + fn dummy_callback(_: &Arc, _: usize) {} // Test acquiring with an alias #[test] fn test_provider_with_alias() { - let mut input = create_config_input(1, "provider1_1", 1, 1); + let mut input = create_config_input(1, &"provider1_1".intern(), 1, 1); let alias = create_config_input_alias(2, "http://alias1", 2, 2); // Adding alias to the provider input.aliases = Some(vec![alias]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); // Create MultiProviderLineup with the provider and alias let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); @@ -993,12 +995,12 @@ mod tests { // Test acquiring from a MultiProviderLineup where the alias has a different priority #[test] fn test_provider_with_priority_alias() { - let mut input = create_config_input(1, "provider2_1", 1, 2); + let mut input = create_config_input(1, &"provider2_1".intern(), 1, 2); let alias = create_config_input_alias(2, "http://alias.com", 0, 2); // Adding alias with different priority input.aliases = Some(vec![alias]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); // The alias has a higher priority, so the alias should be acquired first let rt = tokio::runtime::Runtime::new().unwrap(); @@ -1013,14 +1015,14 @@ mod tests { // Test provider when there are multiple aliases, all with distinct priorities #[test] fn test_provider_with_multiple_aliases() { - let mut input = create_config_input(1, "provider3_1", 1, 1); + let mut input = create_config_input(1, &"provider3_1".intern(), 1, 1); let alias1 = create_config_input_alias(2, "http://alias1.com", 1, 2); let alias2 = create_config_input_alias(3, "http://alias2.com", 0, 1); // Adding multiple aliases input.aliases = Some(vec![alias1, alias2]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1044,14 +1046,14 @@ mod tests { // Test acquiring when all aliases are exhausted #[test] fn test_provider_with_exhausted_aliases() { - let mut input = create_config_input(1, "provider4_1", 1, 1); + let mut input = create_config_input(1, &"provider4_1".intern(), 1, 1); let alias1 = create_config_input_alias(2, "http://alias.com", 2, 1); let alias2 = create_config_input_alias(3, "http://alias.com", -2, 1); // Adding alias input.aliases = Some(vec![alias1, alias2]); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&input, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1077,10 +1079,10 @@ mod tests { // Test acquiring a connection when there is available capacity #[test] fn test_acquire_when_capacity_available() { - let cfg = create_config_input(1, "provider5_1", 1, 2); + let cfg = create_config_input(1, &"provider5_1".intern(), 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); - let connection = get_or_create_provider_connection(&provider_connections, cfg.name.as_str()); + let provider_connections: DashMap, Arc>> = DashMap::new(); + let connection = get_or_create_provider_connection(&provider_connections, &cfg.name); let lineup = SingleProviderLineup::new(&cfg, connection, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1099,10 +1101,10 @@ mod tests { // Test releasing a connection #[test] fn test_release_connection() { - let cfg = create_config_input(1, "provider7_1", 1, 2); + let cfg = create_config_input(1, &"provider7_1".intern(), 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); - let connection = get_or_create_provider_connection(&provider_connections, cfg.name.as_str()); + let provider_connections: DashMap, Arc>> = DashMap::new(); + let connection = get_or_create_provider_connection(&provider_connections, &cfg.name); let lineup = SingleProviderLineup::new(&cfg, connection, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1111,10 +1113,10 @@ mod tests { should_available!(lineup, 1, 5); should_grace_period!(lineup, 1, 5); should_exhausted!(lineup, 5); - lineup.release("provider7_1").await; + lineup.release(&"provider7_1".intern()).await; should_grace_period!(lineup, 1, 5); - lineup.release("provider7_1").await; - lineup.release("provider7_1").await; + lineup.release(&"provider7_1".intern()).await; + lineup.release(&"provider7_1".intern()).await; should_available!(lineup, 1, 5); should_grace_period!(lineup, 1, 5); should_exhausted!(lineup, 5); @@ -1124,7 +1126,7 @@ mod tests { // Test acquiring with MultiProviderLineup and round-robin allocation #[test] fn test_multi_provider_acquire() { - let mut cfg1 = create_config_input(1, "provider8_1", 1, 2); + let mut cfg1 = create_config_input(1, &"provider8_1".intern(), 1, 2); let alias = create_config_input_alias(2, "http://alias1", 1, 1); // Adding alias to the provider @@ -1132,7 +1134,7 @@ mod tests { // Create MultiProviderLineup with the provider and alias let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); + let provider_connections: DashMap, Arc>> = DashMap::new(); let lineup = MultiProviderLineup::new(&cfg1, &provider_connections, &change_callback); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { @@ -1148,9 +1150,9 @@ mod tests { should_grace_period!(lineup, 1, 5); should_grace_period!(lineup, 2, 5); - lineup.release("provider8_1").await; - lineup.release("alias_2").await; - lineup.release("provider8_1").await; + lineup.release(&"provider8_1".intern()).await; + lineup.release(&"alias_2".intern()).await; + lineup.release(&"provider8_1".intern()).await; should_available!(lineup, 1, 5); should_grace_period!(lineup, 1, 5); @@ -1163,10 +1165,10 @@ mod tests { // Test concurrent access to `acquire` using multiple threads #[test] fn test_concurrent_acquire() { - let cfg = create_config_input(1, "provider9_1", 1, 2); + let cfg = create_config_input(1, &"provider9_1".intern(), 1, 2); let change_callback: ProviderConnectionChangeCallback = Arc::new(dummy_callback); - let provider_connections: DashMap>> = DashMap::new(); - let connection = get_or_create_provider_connection(&provider_connections, cfg.name.as_str()); + let provider_connections: DashMap, Arc>> = DashMap::new(); + let connection = get_or_create_provider_connection(&provider_connections, &cfg.name); let lineup = Arc::new(SingleProviderLineup::new(&cfg, connection, &change_callback)); let available_count = Arc::new(AtomicU16::new(2)); diff --git a/backend/src/api/model/stream.rs b/backend/src/api/model/stream.rs index d16c1080a..e452b2f89 100644 --- a/backend/src/api/model/stream.rs +++ b/backend/src/api/model/stream.rs @@ -16,8 +16,8 @@ pub type ProviderStreamResponse = (Option, ProviderStreamIn pub type ProviderStreamFactoryResponse = (BoxedProviderStream, ProviderStreamInfo); -type StreamUrl = String; -type ProviderName = String; +type StreamUrl = Arc; +type ProviderName = Arc; pub enum ProviderStreamState { Custom(ProviderStreamResponse), @@ -28,7 +28,7 @@ pub enum ProviderStreamState { pub struct StreamDetails { pub stream: Option, pub(crate) stream_info: ProviderStreamInfo, - pub provider_name: Option, + pub provider_name: Option>, pub grace_period_millis: u64, pub reconnect_flag: Option>, pub provider_handle: Option, diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/src/api/model/streams/active_client_stream.rs index a7fac73a7..d8079b415 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/src/api/model/streams/active_client_stream.rs @@ -125,8 +125,7 @@ impl ActiveClientStream { let connection_manager = Arc::clone(&app_state.connection_manager); let provider_grace_check = if stream_details.has_grace_period() && stream_details.provider_name.is_some() { - let provider_name = stream_details.provider_name.as_ref().map_or_else(String::new, ToString::to_string); - Some(provider_name) + stream_details.provider_name.clone() } else { None }; diff --git a/backend/src/api/model/streams/shared_stream_manager.rs b/backend/src/api/model/streams/shared_stream_manager.rs index 6ed204b04..595dd6484 100644 --- a/backend/src/api/model/streams/shared_stream_manager.rs +++ b/backend/src/api/model/streams/shared_stream_manager.rs @@ -157,7 +157,7 @@ impl SharedStreamState { } } - async fn subscribe(&self, addr: &SocketAddr, manager: Arc) -> (BoxedProviderStream, Option) { + async fn subscribe(&self, addr: &SocketAddr, manager: Arc) -> (BoxedProviderStream, Option>) { let (client_tx, client_rx) = mpsc::channel(self.buf_size); let mut broadcast_rx = self.broadcaster.subscribe(); let cancel_token = CancellationToken::new(); @@ -428,7 +428,7 @@ impl SharedStreamManager { stream_url: &str, addr: &SocketAddr, manager: Arc, - ) -> Option<(BoxedProviderStream, Option)> { + ) -> Option<(BoxedProviderStream, Option>)> { let shared_state_opt = { let mut shared_streams = self.shared_streams.write().await; if let Some(shared_state) = shared_streams.by_key.get(stream_url).cloned() { @@ -464,7 +464,7 @@ impl SharedStreamManager { addr: &SocketAddr, headers: Vec<(String, String)>, buffer_size: usize, - provider_handle: Option) -> Option<(BoxedProviderStream, Option)> + provider_handle: Option) -> Option<(BoxedProviderStream, Option>)> where S: Stream> + Unpin + 'static + Send, E: std::fmt::Debug + Send, @@ -487,7 +487,7 @@ impl SharedStreamManager { app_state: &AppState, stream_url: &str, addr: &SocketAddr, - ) -> Option<(BoxedProviderStream, Option)> { + ) -> Option<(BoxedProviderStream, Option>)> { let manager = Arc::clone(&app_state.shared_stream_manager); if let Some(result) = app_state.shared_stream_manager.subscribe_stream(stream_url, addr, manager).await { app_state.active_provider.add_shared_connection(addr, stream_url).await; diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index 1e948ba60..88401697c 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use crate::api::config_file::ConfigFile; use crate::api::model::AppState; use crate::model::{is_input_expired, ConfigInput, PanelApiConfig, PanelApiQueryParam}; @@ -7,14 +8,14 @@ use log::{debug, error, warn}; use serde_json::Value; use shared::error::{info_err_res, info_err, TuliproxError}; use shared::model::{ConfigInputAliasDto, InputType}; -use shared::utils::{get_credentials_from_url, parse_timestamp, sanitize_sensitive_info, trim_last_slash}; +use shared::utils::{get_credentials_from_url, parse_timestamp, sanitize_sensitive_info, trim_last_slash, Internable}; use std::path::{Path, PathBuf}; use std::sync::Arc; use url::Url; #[derive(Debug, Clone)] struct AccountCredentials { - name: String, + name: Arc, username: String, password: String, exp_date: Option, @@ -293,8 +294,8 @@ fn collect_expired_accounts(input: &ConfigInput) -> Vec { async fn patch_source_yml_add_alias( app_state: &Arc, source_file_path: &Path, - input_name: &str, - alias_name: &str, + input_name: &Arc, + alias_name: &Arc, base_url: &str, username: &str, password: &str, @@ -305,13 +306,13 @@ async fn patch_source_yml_add_alias( Err(e) => return info_err_res!("panel_api: failed to read source file: {e}"), }; - let Some(input) = sources.inputs.iter_mut().find(|i| i.name == input_name) else { + let Some(input) = sources.inputs.iter_mut().find(|i| &i.name == input_name) else { return info_err_res!("panel_api: could not find input '{input_name}' in source.yml"); }; let alias = ConfigInputAliasDto { id: 0, - name: alias_name.to_string(), + name: Arc::clone(alias_name), url: base_url.to_string(), username: Some(username.to_string()), password: Some(password.to_string()), @@ -329,7 +330,7 @@ async fn patch_source_yml_add_alias( async fn patch_source_yml_update_exp_date( app_state: &Arc, source_file_path: &Path, - input_name: &str, + input_name: &Arc, account_name: &str, exp_date: i64, ) -> Result<(), TuliproxError> { @@ -338,7 +339,7 @@ async fn patch_source_yml_update_exp_date( Err(e) => return info_err_res!("panel_api: failed to read source file: {e}"), }; - let Some(input) = sources.inputs.iter_mut().find(|i| i.name == input_name) else { + let Some(input) = sources.inputs.iter_mut().find(|i| &i.name == input_name) else { return info_err_res!("panel_api: could not find input '{input_name}' in source.yml"); }; @@ -471,14 +472,14 @@ async fn patch_batch_csv_update_exp_date( const MAX_ALIAS_NAME_ATTEMPTS: usize = 1000; -fn derive_unique_alias_name(existing: &[String], input_name: &str, username: &str) -> String { +fn derive_unique_alias_name(existing: &HashSet<&str>, input_name: &Arc, username: &str) -> String { let base = format!("{input_name}-{username}"); - if !existing.contains(&base) { + if !existing.contains(base.as_str()) { return base; } for i in 2..MAX_ALIAS_NAME_ATTEMPTS { let cand = format!("{base}-{i}"); - if !existing.contains(&cand) { + if !existing.contains(cand.as_str()) { return cand; } } @@ -548,11 +549,12 @@ async fn try_create_new_account( let base_url = base_url_from_resp.unwrap_or_else(|| input.url.clone()); let base_url = extract_base_url(base_url.as_str()).unwrap_or_else(|| base_url.clone()); - let mut existing_names: Vec = vec![input.name.clone()]; + let mut existing_names: HashSet<&str> = HashSet::new(); + existing_names.insert(&*input.name); if let Some(aliases) = input.aliases.as_ref() { - existing_names.extend(aliases.iter().map(|a| a.name.clone())); + existing_names.extend(aliases.iter().map(|a| &*a.name)); } - let alias_name = derive_unique_alias_name(&existing_names, &input.name, &username); + let alias_name = derive_unique_alias_name(&existing_names, &input.name, &username).intern(); let exp_date = panel_client_info(app_state, panel_cfg, &username, &password) .await diff --git a/backend/src/api/scheduler.rs b/backend/src/api/scheduler.rs index 011234214..716ebd065 100644 --- a/backend/src/api/scheduler.rs +++ b/backend/src/api/scheduler.rs @@ -4,11 +4,12 @@ use crate::processing::processor::playlist::exec_processing; use crate::utils::exit; use chrono::{DateTime, FixedOffset, Local}; use cron::Schedule; -use log::error; +use log::{error}; use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use tokio_util::sync::CancellationToken; +use shared::utils::interner_gc; pub fn datetime_to_instant(datetime: DateTime) -> Instant { // Convert DateTime to SystemTime @@ -105,6 +106,24 @@ fn get_process_targets(cfg: &Arc, process_targets: &Arc) { + let app_state = Arc::clone(app_state); + tokio::spawn({ + async move { + loop { + tokio::time::sleep(Duration::from_secs(180)).await; + if let Some(permit) = app_state.update_guard.try_playlist() { + // Gate check: ensure updates aren't in progress; permit dropped to allow concurrent updates during GC + drop(permit); + interner_gc(); + } + } + } + }); +} + #[cfg(test)] mod tests { use crate::api::scheduler::datetime_to_instant; diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index e1c4038c4..3c45213c9 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -128,7 +128,7 @@ mod tests { use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; use std::path::PathBuf; use std::time::Duration; - use shared::utils::intern; + use shared::utils::Internable; use crate::library::{MediaClassification, MediaClassifier}; fn create_test_config(tmdb_enabled: bool) -> LibraryConfig { @@ -158,8 +158,8 @@ mod tests { formats: vec![], }, playlist: LibraryPlaylistConfig { - movie_category: intern("Movies"), - series_category: intern("Series"), + movie_category: "Movies".intern(), + series_category: "Series".intern(), }, } } diff --git a/backend/src/library/scanner.rs b/backend/src/library/scanner.rs index 5bc88a62a..55e74440f 100644 --- a/backend/src/library/scanner.rs +++ b/backend/src/library/scanner.rs @@ -281,8 +281,8 @@ impl LibraryScanner { #[cfg(test)] mod tests { + use shared::utils::Internable; use super::*; - use shared::utils::intern; use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig}; fn create_test_config() -> LibraryConfig { @@ -312,8 +312,8 @@ mod tests { formats: vec![], }, playlist: LibraryPlaylistConfig { - movie_category: intern("Local Movies"), - series_category: intern("Local Series"), + movie_category: "Local Movies".intern(), + series_category: "Local Series".intern(), }, } } diff --git a/backend/src/model/config/app.rs b/backend/src/model/config/app.rs index 0fd4def99..23af47845 100644 --- a/backend/src/model/config/app.rs +++ b/backend/src/model/config/app.rs @@ -201,20 +201,20 @@ impl AppConfig { self.api_proxy.load().as_ref().as_ref().and_then(|api_proxy| api_proxy.get_user_credentials(username)) } - pub fn get_input_by_name(&self, input_name: &str) -> Option> { + pub fn get_input_by_name(&self, input_name: &Arc) -> Option> { let sources = > as Access>::load(&self.sources); for input in &sources.inputs { - if input.name == input_name { + if &input.name == input_name { return Some(Arc::clone(input)); } } None } - pub fn get_input_options_by_name(&self, input_name: &str) -> Option { + pub fn get_input_options_by_name(&self, input_name: &Arc) -> Option { let sources = > as Access>::load(&self.sources); for input in &sources.inputs { - if input.name == input_name { + if &input.name == input_name { return input.options.clone(); } } diff --git a/backend/src/model/config/input.rs b/backend/src/model/config/input.rs index daeee59b0..7d25ed9ff 100644 --- a/backend/src/model/config/input.rs +++ b/backend/src/model/config/input.rs @@ -4,12 +4,13 @@ use chrono::Utc; use log::warn; use shared::error::TuliproxError; use shared::model::{ConfigInputAliasDto, ConfigInputDto, ConfigInputOptionsDto, InputFetchMethod, InputType, StagedInputDto}; -use shared::utils::{get_credentials_from_url}; +use shared::utils::{get_credentials_from_url, Internable}; use shared::{check_input_connections, info_err_res, write_if_some}; use shared::check_input_credentials; use std::collections::HashMap; use std::fmt; use std::path::PathBuf; +use std::sync::Arc; use url::Url; use crate::model::config::panel_api::PanelApiConfig; @@ -71,7 +72,7 @@ impl InputUserInfo { #[derive(Debug, Clone, Default)] pub struct StagedInput { - pub name: String, + pub name: Arc, pub url: String, pub username: Option, pub password: Option, @@ -98,7 +99,7 @@ impl From<&StagedInputDto> for StagedInput { #[derive(Debug, Clone)] pub struct ConfigInputAlias { pub id: u16, - pub name: String, + pub name: Arc, pub url: String, pub username: Option, pub password: Option, @@ -126,7 +127,7 @@ impl From<&ConfigInputAliasDto> for ConfigInputAlias { #[derive(Debug, Clone, Default)] pub struct ConfigInput { pub id: u16, - pub name: String, + pub name: Arc, pub input_type: InputType, pub headers: HashMap, pub url: String, @@ -150,7 +151,7 @@ pub struct ConfigInput { impl ConfigInput { pub fn prepare(&mut self) -> Result, TuliproxError> { let batch_file_path = self.prepare_batch(); - self.name = self.name.trim().to_string(); + self.name = self.name.trim().intern(); check_input_credentials!(self, self.input_type, false, false); check_input_connections!(self, self.input_type, false); if let Some(staged_input) = &mut self.staged { diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index 1c6b5ce6e..48300bf20 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -3,7 +3,7 @@ use shared::error::{TuliproxError, info_err_res}; use shared::model::{LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; use std::path::PathBuf; use std::sync::Arc; -use shared::utils::{default_metadata_path, intern}; +use shared::utils::{default_metadata_path, Internable}; #[derive(Debug, Clone, Default)] pub struct LibraryScanDirectory { @@ -121,8 +121,8 @@ impl From<&LibraryConfigDto> for LibraryConfig { formats: dto.metadata.formats.clone(), }, playlist: LibraryPlaylistConfig { - movie_category: intern(&dto.playlist.movie_category), - series_category: intern(&dto.playlist.series_category), + movie_category: dto.playlist.movie_category.as_str().intern(), + series_category: dto.playlist.series_category.as_str().intern(), }, } } diff --git a/backend/src/model/config/source.rs b/backend/src/model/config/source.rs index 5272b3755..8182895b4 100644 --- a/backend/src/model/config/source.rs +++ b/backend/src/model/config/source.rs @@ -8,7 +8,7 @@ use std::sync::Arc; #[derive(Debug, Clone)] pub struct ConfigSource { - pub inputs: Vec, + pub inputs: Vec>, pub targets: Vec>, } @@ -98,7 +98,7 @@ impl SourcesConfig { None } - pub fn get_source_inputs_by_target_by_name(&self, target_name: &str) -> Option> { + pub fn get_source_inputs_by_target_by_name(&self, target_name: &str) -> Option>> { for source in &self.sources { for target in &source.targets { if target.name == target_name { @@ -176,7 +176,7 @@ impl SourcesConfig { file_names } - pub fn get_input_by_name(&self, name: &str) -> Option<&Arc> { - self.inputs.iter().find(|i| i.name == name) + pub fn get_input_by_name(&self, name: &Arc) -> Option<&Arc> { + self.inputs.iter().find(|i| &i.name == name) } } diff --git a/backend/src/model/config/trakt.rs b/backend/src/model/config/trakt.rs index e60406654..79ea8b6e5 100644 --- a/backend/src/model/config/trakt.rs +++ b/backend/src/model/config/trakt.rs @@ -69,6 +69,7 @@ impl From<&TraktListConfig> for TraktListConfigDto { #[derive(Debug, Clone)] pub struct TraktConfig { + pub enabled: bool, pub api: TraktApiConfig, pub lists: Vec, } @@ -77,6 +78,7 @@ macros::from_impl!(TraktConfig); impl From<&TraktConfigDto> for TraktConfig { fn from(dto: &TraktConfigDto) -> Self { Self { + enabled: dto.enabled, api: TraktApiConfig::from(&dto.api), lists: dto.lists.iter().map(Into::into).collect(), } @@ -85,6 +87,7 @@ impl From<&TraktConfigDto> for TraktConfig { impl From<&TraktConfig> for TraktConfigDto { fn from(dto: &TraktConfig) -> Self { Self { + enabled: dto.enabled, api: TraktApiConfigDto::from(&dto.api), lists: dto.lists.iter().map(TraktListConfigDto::from).collect(), } diff --git a/backend/src/model/input_source.rs b/backend/src/model/input_source.rs index 8fe3c0787..fc5daab63 100644 --- a/backend/src/model/input_source.rs +++ b/backend/src/model/input_source.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; +use std::sync::Arc; use shared::model::InputFetchMethod; use crate::model::{ConfigInput, StagedInput}; #[derive(Clone, Debug)] pub struct InputSource { - pub name: String, + pub name: Arc, pub url: String, pub username: Option, pub password: Option, diff --git a/backend/src/model/playlist.rs b/backend/src/model/playlist.rs index e1a2204b7..1bb6f5fa4 100644 --- a/backend/src/model/playlist.rs +++ b/backend/src/model/playlist.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use crate::model::{ConfigInput, TVGuide}; -use shared::model::{PlaylistGroup, PlaylistItem, UUIDType}; +use shared::model::{PlaylistGroup, PlaylistItem}; +use shared::model::UUIDType; use crate::repository::PlaylistSource; pub struct FetchedPlaylist<'a> { diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index cb1148865..4c3bde2de 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use futures::TryFutureExt; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use url::Url; -use shared::utils::sanitize_sensitive_info; +use shared::utils::{sanitize_sensitive_info, Internable}; use crate::api::model::AppState; use crate::model::{InputSource}; use crate::utils::async_file_reader; @@ -227,7 +227,7 @@ pub async fn parse_xmltv_for_web_ui_from_url(app_state: &Arc, url: &st if let Ok(request_url) = Url::parse(url) { let client = app_state.http_client.load(); let input_source: InputSource = InputSource { - name: String::from("xmltv"), + name: "xmltv".intern(), url: request_url.to_string(), username: None, password: None, diff --git a/backend/src/model/xtream.rs b/backend/src/model/xtream.rs index 122b0039e..316941f93 100644 --- a/backend/src/model/xtream.rs +++ b/backend/src/model/xtream.rs @@ -1,9 +1,10 @@ +use std::sync::Arc; use crate::model::{AppConfig, ProxyUserCredentials}; use crate::model::{ConfigTarget, XtreamTargetOutput}; use serde::{Deserialize, Serialize}; use serde_json::Value; use shared::model::{PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, ProxyUserStatus, XtreamMappingOptions}; -use shared::utils::{deserialize_as_string, deserialize_number_from_string_or_zero}; +use shared::utils::{arc_str_serde, deserialize_number_from_string_or_zero}; use enum_iterator::all; #[derive(Debug, Default)] @@ -16,8 +17,8 @@ pub struct XtreamLoginInfo { pub struct XtreamCategory { #[serde(deserialize_with = "deserialize_number_from_string_or_zero", serialize_with = "shared::utils::serialize_number_as_string")] pub category_id: u32, - #[serde(deserialize_with = "deserialize_as_string")] - pub category_name: String, + #[serde(with = "arc_str_serde")] + pub category_name: Arc, //pub parent_id: i32, #[serde(default)] pub channels: Vec, @@ -79,7 +80,7 @@ pub fn normalize_release_date(document: &mut serde_json::Map) { #[derive(Deserialize, Serialize, Clone)] pub struct PlaylistXtreamCategory { #[serde(alias = "category_id")] - pub id: String, + pub id: u32, #[serde(alias = "category_name")] pub name: String, } \ No newline at end of file diff --git a/backend/src/processing/parser/hls.rs b/backend/src/processing/parser/hls.rs index c0fc5680b..408b5d0a4 100644 --- a/backend/src/processing/parser/hls.rs +++ b/backend/src/processing/parser/hls.rs @@ -1,12 +1,16 @@ +use std::borrow::Cow; use crate::model::ProxyUserCredentials; -use shared::utils::{CONSTANTS, HLS_PREFIX}; use crate::utils::{deobfuscate_text, obfuscate_text}; +use shared::utils::{CONSTANTS, HLS_PREFIX}; use std::str; +use url::Url; +use shared::concat_string; const TOKEN_SEPARATOR: char = '\x1F'; +const TOKEN_SEPARATOR_STR: &str = "\x1F"; fn create_hls_session_token_and_url(secret: &[u8], session_token: &str, stream_url: &str) -> Option { - if let Ok(cookie_value) = obfuscate_text(secret, &format!("{session_token}{TOKEN_SEPARATOR}{stream_url}")) { + if let Ok(cookie_value) = obfuscate_text(secret, &concat_string!(session_token, TOKEN_SEPARATOR_STR, stream_url)) { return Some(cookie_value); } None @@ -15,7 +19,7 @@ fn create_hls_session_token_and_url(secret: &[u8], session_token: &str, stream_u pub fn get_hls_session_token_and_url_from_token(secret: &[u8], token: &str) -> Option<(Option, String)> { if let Ok(decrypted) = deobfuscate_text(secret, token) { let parts: Vec<&str> = decrypted.split(TOKEN_SEPARATOR).collect(); - if !parts.is_empty() && parts.len() == 2 { + if parts.len() == 2 { let session_token: String = parts[0].to_string(); let stream_url: String = parts[1].to_string(); return Some((Some(session_token), stream_url)); @@ -24,7 +28,6 @@ pub fn get_hls_session_token_and_url_from_token(secret: &[u8], token: &str) -> O None } - pub struct RewriteHlsProps<'a> { pub secret: &'a [u8; 16], pub base_url: &'a str, @@ -35,33 +38,43 @@ pub struct RewriteHlsProps<'a> { pub user_token: Option<&'a str>, } -fn rewrite_hls_url(input: &str, replacement: &str) -> String { - if replacement.starts_with('/') { - let parts = input.splitn(4, '/').collect::>(); - if parts.len() < 4 { - return replacement.to_string(); // less than 3 Slashes → replace all - } - format!("{}/{}/{}{}", parts[0], parts[1], parts[2], replacement) - } else { - match input.rsplitn(2, '/').collect::>().as_slice() { - [_after, before] => format!("{before}/{replacement}"), - [_only] => replacement.to_string(), - _ => input.to_string(), - } +/// Rewrites an HLS URI relative to a base playlist URL. +/// Absolute URIs are returned unchanged. +pub fn rewrite_hls_url<'a>(base: &'a str, reference: &'a str) -> Cow<'a, str> { + // absolute URI → passthrough + if Url::parse(reference).is_ok() { + return Cow::Borrowed(reference); } + + let Ok(base_url) = Url::parse(base) else { + return Cow::Borrowed(reference); + }; + + base_url.join(reference).map_or_else(|_| Cow::Borrowed(reference), |u| Cow::Owned(u.to_string())) } -fn rewrite_uri_attrib(line: &str, props: &RewriteHlsProps) -> String { - if let Some(caps) = CONSTANTS.re_hls_uri.captures(line) { - let uri = &caps[1]; - let target_url = &rewrite_hls_url(&props.hls_url, uri); - if let Some(user_token) = &props.user_token { - if let Some(token) = create_hls_session_token_and_url(props.secret, user_token, target_url) { - return CONSTANTS.re_hls_uri.replace(line, format!(r#"URI="{token}""#)).to_string(); - } - } - } - line.to_string() +fn rewrite_uri_attrib<'a>(line: &'a str, props: &RewriteHlsProps) -> Cow<'a, str> { + let Some(caps) = CONSTANTS.re_hls_uri.captures(line) else { + return Cow::Borrowed(line); + }; + + let uri = &caps[1]; + let rewritten = rewrite_hls_url(&props.hls_url, uri); + + let final_uri = if let Some(user_token) = &props.user_token { + create_hls_session_token_and_url( + props.secret, + user_token, + &rewritten, + ).map(Cow::Owned).unwrap_or(rewritten) + } else { + rewritten + }; + + Cow::Owned(CONSTANTS + .re_hls_uri + .replace(line, format!(r#"URI="{final_uri}""#)) + .to_string()) } pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> String { @@ -69,7 +82,6 @@ pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> Stri let password = &user.password; let mut result = Vec::new(); for line in props.content.lines() { - if line.trim().is_empty() { continue; } @@ -77,16 +89,12 @@ pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> Stri // skip comments if line.starts_with('#') { let rewritten = rewrite_uri_attrib(line, props); - result.push(rewritten); + result.push(rewritten.to_string()); continue; } // target url - let target_url = if line.starts_with("http") { - line.to_string() - } else { - rewrite_hls_url(&props.hls_url, line) - }; + let target_url = rewrite_hls_url(&props.hls_url, line); if let Some(user_token) = &props.user_token { if let Some(token) = create_hls_session_token_and_url(props.secret, user_token, &target_url) { let url = format!( @@ -110,6 +118,7 @@ pub fn rewrite_hls(user: &ProxyUserCredentials, props: &RewriteHlsProps) -> Stri mod test { use rand::RngCore; use shared::utils::u32_to_base64; + use crate::processing::parser::hls::{rewrite_hls_url}; #[test] fn test_token_size() { @@ -119,4 +128,75 @@ mod test { } } + #[test] + fn rewrite_http_relative_segment() { + let base = "http://example.com/hls/playlist.m3u8"; + let uri = "seg001.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "http://example.com/hls/seg001.ts"); + } + + #[test] + fn rewrite_http_root_relative_segment() { + let base = "http://example.com/hls/playlist.m3u8"; + let uri = "/media/seg001.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "http://example.com/media/seg001.ts"); + } + + #[test] + fn rewrite_http_parent_directory() { + let base = "http://example.com/hls/level1/playlist.m3u8"; + let uri = "../seg001.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "http://example.com/hls/seg001.ts"); + } + + #[test] + fn rewrite_https_absolute_passthrough() { + let base = "http://example.com/hls/playlist.m3u8"; + let uri = "https://cdn.example.org/video/seg.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, uri); + } + + #[test] + fn rewrite_file_relative_segment() { + let base = "file:///mnt/media/hls/playlist.m3u8"; + let uri = "seg001.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "file:///mnt/media/hls/seg001.ts"); + } + + #[test] + fn rewrite_file_parent_directory() { + let base = "file:///mnt/media/hls/level1/playlist.m3u8"; + let uri = "../seg001.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "file:///mnt/media/hls/seg001.ts"); + } + + #[test] + fn rewrite_file_absolute_passthrough() { + let base = "file:///mnt/media/hls/playlist.m3u8"; + let uri = "file:///mnt/other/seg.ts"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, uri); + } + + #[test] + fn rewrite_hls_fragment() { + let base = "http://example.com/hls/playlist.m3u8"; + let fragment = "seg.ts#t=10"; + + let out = rewrite_hls_url(base, fragment); + assert_eq!(out, "http://example.com/hls/seg.ts#t=10"); + } } \ No newline at end of file diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 902e17954..0c815e31b 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -1,25 +1,26 @@ use crate::model::{Config, ConfigInput}; use crate::utils::request::DynReader; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use shared::utils::{default_supported_video_extensions, extract_id_from_url, StringInterner}; +use shared::utils::{default_supported_video_extensions, extract_id_from_url, Internable}; use std::borrow::BorrowMut; +use std::sync::Arc; use tokio::io::AsyncBufReadExt; use indexmap::IndexMap; use shared::concat_string; // other implementations like calculating text_distance on all titles took too much time // we keep it now as simple as possible and less memory intensive. -fn get_title_group(text: &str) -> String { +fn get_title_group(text: &Arc) -> Arc { let alphabetic_only: String = text.chars().map(|c| if c.is_alphanumeric() { c } else { ' ' }).collect(); let parts = alphabetic_only.split_whitespace(); let mut combination = String::new(); for p in parts { combination = concat_string!(&combination, " " , p); if combination.len() > 2 { - return combination; + return combination.intern(); } } - text.to_string() + text.clone() } #[inline] @@ -90,11 +91,11 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { } } -fn create_empty_playlistitem_header(interner: &mut StringInterner, input_name: &str, url: String) -> PlaylistItemHeader { +fn create_empty_playlistitem_header(input_name: &Arc, url: String) -> PlaylistItemHeader { PlaylistItemHeader { - url, + url: Arc::from(url), category_id: 0, - input_name: interner.intern(input_name), + input_name: Arc::clone(input_name), ..Default::default() } } @@ -110,7 +111,7 @@ macro_rules! process_header_fields { }; } -fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixes: &[String], content: &str, url: String) -> PlaylistItemHeader { +fn process_header(input_name: &Arc, video_suffixes: &[String], content: &str, url: String) -> PlaylistItemHeader { let url_id = extract_id_from_url(&url); let url_types = if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { // TODO find Series based on group or configured names @@ -119,7 +120,7 @@ fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixe None }; - let mut plih = create_empty_playlistitem_header(interner, input_name, url); + let mut plih = create_empty_playlistitem_header(input_name, url); let mut it = content.chars(); let mut stack = String::with_capacity(64); let line_token = token_till(&mut stack, &mut it, ':', false); @@ -133,7 +134,7 @@ fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixe if chr.is_whitespace() { // skip } else if chr == ',' { - plih.title = get_value(&mut stack, &mut it); + plih.title = get_value(&mut stack, &mut it).intern(); } else { stack.push(chr); let token = token_till(&mut stack, &mut it, '=', true); @@ -146,19 +147,22 @@ fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixe } } else if token == "tvg-chno" { plih.chno = value.parse::().unwrap_or(0); + } else if token == "group-title" { + plih.group = value.intern(); + } else if token == "tvg-id" { + plih.id = value.intern(); + } else if token == "tvg-name" { + plih.name = value.intern(); + } else if token == "tvg-logo" { + plih.logo = value.intern(); + } else if token == "tvg-logo-small" { + plih.logo_small = value.intern(); } else { process_header_fields!(plih, token.as_str(), - (id, "tvg-id"), - (name, "tvg-name"), (parent_code, "parent-code"), (audio_track, "audio-track"), - (logo, "tvg-logo"), - (logo_small, "tvg-logo-small"), (time_shift, "timeshift"), - (rec, "tvg-rec"); value.clone()); - if token == "group-title" { - plih.group = interner.intern(&value); - } + (rec, "tvg-rec"); value.intern()); } } } @@ -170,14 +174,14 @@ fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixe if plih.id.is_empty() { plih.epg_channel_id = None; if let Some(pid) = provider_id { - plih.id = pid; + plih.id = pid.intern(); } else if let Some(chanid) = url_id { - plih.id = chanid; + plih.id = chanid.intern(); } } else { - plih.epg_channel_id = Some(plih.id.clone()); + plih.epg_channel_id = Some(Arc::clone(&plih.id)); if let Some(pid) = provider_id { - plih.id = pid; + plih.id = pid.intern(); } } } @@ -204,8 +208,7 @@ fn process_header(interner: &mut StringInterner, input_name: &str, video_suffixe pub async fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader, mut visit: F) { let mut header: Option = None; let mut group: Option = None; - let mut interner = StringInterner::new(); - let input_name = input.name.as_str(); + let input_name = &input.name; let video_suffixes = match cfg.video.as_ref() { Some(config) => { @@ -228,19 +231,19 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp continue; } if let Some(header_value) = header { - let mut item = PlaylistItem { header: process_header(&mut interner, input_name, &video_suffixes, &header_value, line) }; + let mut item = PlaylistItem { header: process_header(input_name, &video_suffixes, &header_value, line) }; let header = &mut item.header; header.source_ordinal = ord_counter; ord_counter += 1; - if header.group.is_empty() { - if let Some(group_value) = group { - header.group = interner.intern(&group_value); - } else { - let current_title = header.title.clone(); - header.group = interner.intern(&get_title_group(current_title.as_str())); + if header.group.is_empty() { + if let Some(group_value) = group { + header.group = group_value.intern(); + } else { + let group = get_title_group(&header.title); + header.group = group; + } } - } - visit(item); + visit(item); } header = None; group = None; @@ -274,53 +277,50 @@ pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> V #[cfg(test)] mod test { + use shared::utils::Internable; use crate::processing::parser::m3u::process_header; - use shared::utils::StringInterner; #[test] fn test_process_header_1() { - let mut interner = StringInterner::new(); - let input: &str = "hello"; + let input = "hello".intern(); let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney" , Seven"#; - let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); - assert_eq!(pli.title, "Seven"); - assert_eq!(pli.id, "abc-seven"); - assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); + let pli = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(pli.title, "Seven".intern()); + assert_eq!(pli.id, "abc-seven".intern()); + assert_eq!(pli.logo, "https://abc.nz/.images/seven.png".intern()); assert_eq!(pli.chno, 7); assert_eq!(&*pli.group, "Sydney"); } #[test] fn test_process_header_2() { - let mut interner = StringInterner::new(); - let input: &str = "hello"; + let input = "hello".intern(); let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney", Seven"#; - let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); - assert_eq!(pli.title, "Seven"); - assert_eq!(pli.id, "abc-seven"); - assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); + let pli = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(pli.title, "Seven".intern()); + assert_eq!(pli.id, "abc-seven".intern()); + assert_eq!(pli.logo, "https://abc.nz/.images/seven.png".intern()); assert_eq!(pli.chno, 7); assert_eq!(&*pli.group, "Sydney"); } #[test] fn test_process_header_xui_id() { - let mut interner = StringInterner::new(); - let input: &str = "hello"; + let input = "hello".intern(); let video_suffixes = Vec::new(); let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 tvg-id="abc-seven" xui-id="provider-123" group-title="Sydney", Seven"#; - let pli = process_header(&mut interner, input, &video_suffixes, line, url.to_string()); - assert_eq!(pli.title, "Seven"); - assert_eq!(pli.id, "provider-123"); // Should use xui-id - assert_eq!(pli.epg_channel_id, Some("abc-seven".to_string())); // Should preserve original tvg-id + let pli = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(pli.title, "Seven".intern()); + assert_eq!(pli.id, "provider-123".intern()); // Should use xui-id + assert_eq!(pli.epg_channel_id, Some("abc-seven".intern())); // Should preserve original tvg-id assert_eq!(&*pli.group, "Sydney"); } } \ No newline at end of file diff --git a/backend/src/processing/parser/xmltv.rs b/backend/src/processing/parser/xmltv.rs index b90bf74ad..a7a230b55 100644 --- a/backend/src/processing/parser/xmltv.rs +++ b/backend/src/processing/parser/xmltv.rs @@ -7,7 +7,7 @@ use dashmap::DashMap; use quick_xml::events::{BytesStart, BytesText, Event}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use shared::model::EpgNamePrefix; -use shared::utils::{deunicode_string, CONSTANTS}; +use shared::utils::{deunicode_string, Internable, CONSTANTS}; use std::borrow::Cow; use std::cell::RefCell; use std::cmp::min; @@ -488,7 +488,7 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option { let epg_children: Mutex>> = Mutex::new(Vec::new()); let epg_attributes: Option, String>> = tv_guides.first().and_then(|t| t.attributes.clone()); let count = tv_guides.iter().map(|tvg| tvg.children.len()).sum(); - let channel_mapping: DashMap = DashMap::with_capacity(count); + let channel_mapping: DashMap, i16> = DashMap::with_capacity(count); let mut sorted_guides = tv_guides.to_vec(); // sort by priority @@ -499,14 +499,15 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option { guide.children.iter().for_each(|c| { if c.name.as_ref() == EPG_TAG_CHANNEL { if let Some(chan_id) = c.get_attribute_value(EPG_ATTRIB_ID) { + let chan_id = chan_id.intern(); let should_add = { // if not stored - !channel_mapping.contains_key(chan_id) || + !channel_mapping.contains_key(&chan_id) || // or if priority is higher (less means higher priority) - channel_mapping.get(chan_id).as_deref().is_none_or(|&priority| guide.priority < priority) + channel_mapping.get(&chan_id).as_deref().is_none_or(|&priority| guide.priority < priority) }; if should_add { - if let Some(mut existing) = channel_mapping.get_mut(chan_id) { + if let Some(mut existing) = channel_mapping.get_mut(&chan_id) { if guide.priority < *existing { *existing = guide.priority; children.push(c.clone()); @@ -522,7 +523,8 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option { guide.children.iter().for_each(|c| { if c.name.as_ref() == EPG_TAG_PROGRAMME { if let Some(chan_id) = c.get_attribute_value(EPG_ATTRIB_CHANNEL) { - if let Some(stored_priority) = channel_mapping.get(chan_id) { + let chan_id = chan_id.intern(); + if let Some(stored_priority) = channel_mapping.get(&chan_id) { if *stored_priority == guide.priority { children.push(c.clone()); } diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 9a6650ec0..cdbd58f1c 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -1,19 +1,18 @@ -use std::borrow::Cow; use crate::model::ConfigInput; -use crate::model::{XtreamCategory}; +use crate::model::XtreamCategory; use crate::utils::request::DynReader; use crate::utils::xtream::get_xtream_stream_url_base; -use shared::error::{notify_err_res, notify_err, TuliproxError}; +use indexmap::IndexMap; +use serde::Deserializer; +use shared::error::{notify_err, notify_err_res, TuliproxError}; +use shared::model::UUIDType; use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, - SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, + SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{generate_playlist_uuid, trim_last_slash, StringInterner}; -use indexmap::IndexMap; +use shared::utils::{generate_playlist_uuid, trim_last_slash, Internable}; +use std::sync::Arc; use tokio::task::spawn_blocking; -use serde::Deserializer; - - async fn map_to_xtream_category(categories: DynReader) -> Result, TuliproxError> { spawn_blocking(move || { @@ -51,46 +50,46 @@ async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader }).await.map_err(|e| notify_err!("Mapping xtream streams failed: {e}"))? } -fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, password: &'a str, episode: &'a SeriesStreamDetailEpisodeProperties) -> Cow<'a, str> { +fn create_xtream_series_episode_url(url: &str, username: &str, password: &str, episode: &SeriesStreamDetailEpisodeProperties) -> Arc { if episode.direct_source.is_empty() { let ext = episode.container_extension.clone(); let stream_base_url = format!("{url}/series/{username}/{password}/{}.{ext}", episode.id); - Cow::Owned(stream_base_url) + stream_base_url.into() } else { - Cow::Borrowed(&episode.direct_source) + Arc::clone(&episode.direct_source) } } - pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, - group_title: &str, series_name: &str, input: &ConfigInput, - interner: &mut StringInterner) -> Option> { +pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, + group_title: &str, series_name: &Arc, input: &ConfigInput) -> Option> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); if let Some(episodes) = series_info.details.as_ref().and_then(|d| d.episodes.as_ref()) { let result: Vec = episodes.iter().map(|episode| { + let episode_id = episode.id.to_string(); let episode_url = create_xtream_series_episode_url(url, username, password, episode); let episode_info = EpisodeStreamProperties::from_series(series_info, episode); - PlaylistItem { - header: PlaylistItemHeader { - id: episode.id.to_string(), - uuid: generate_playlist_uuid(&input.name, &episode.id.to_string(), PlaylistItemType::Series, &episode_url), - // we use parent_code to track the parent series - parent_code: parent_uuid.to_string(), - name: series_name.to_string(), - logo: episode.movie_image.clone(), - group: interner.intern(group_title), - title: episode.title.clone(), - url: episode_url.to_string(), - item_type: PlaylistItemType::Series, - xtream_cluster: XtreamCluster::Series, - additional_properties: Some(StreamProperties::Episode(episode_info)), - category_id: 0, - input_name: interner.intern(&input.name), - ..Default::default() - } - } + PlaylistItem { + header: PlaylistItemHeader { + uuid: generate_playlist_uuid(&input.name, &episode_id, PlaylistItemType::Series, &episode_url), + id: episode_id.into(), + // we use parent_code to track the parent series + parent_code: parent_uuid.intern(), + name: Arc::clone(series_name), + logo: Arc::clone(&episode.movie_image), + group: group_title.intern(), + title: Arc::clone(&episode.title), + url: episode_url, + item_type: PlaylistItemType::Series, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Episode(episode_info)), + category_id: 0, + input_name: input.name.intern(), + ..Default::default() + } + } }).collect(); return if result.is_empty() { None } else { Some(result) }; } @@ -122,12 +121,12 @@ pub fn get_xtream_url(xtream_cluster: XtreamCluster, url: &str, stream_base_url } -pub fn create_xtream_url<'a>(xtream_cluster: XtreamCluster, url: &'a str, username: &'a str, password: &'a str, - stream: &'a StreamProperties, live_stream_use_prefix: bool, live_stream_without_extension: bool) -> Cow<'a, str> { +pub fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &str, password: &str, + stream: &StreamProperties, live_stream_use_prefix: bool, live_stream_without_extension: bool) -> Arc { stream.get_direct_source().unwrap_or_else(|| - Cow::Owned(get_xtream_url(xtream_cluster, url, username, password, stream.get_stream_id(), + get_xtream_url(xtream_cluster, url, username, password, stream.get_stream_id(), stream.get_container_extension().as_deref(), - live_stream_use_prefix, live_stream_without_extension)) + live_stream_use_prefix, live_stream_without_extension).into() ) } @@ -137,7 +136,6 @@ pub async fn parse_xtream(input: &ConfigInput, streams: DynReader) -> Result>, TuliproxError> { match map_to_xtream_category(categories).await { Ok(xtream_categories) => { - let mut interner = StringInterner::new(); let input_name = input.name.clone(); let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); @@ -149,7 +147,7 @@ pub async fn parse_xtream(input: &ConfigInput, xtream_categories.into_iter().map(|category| (category.category_id, category)).collect(); let mut unknown_grp = XtreamCategory { category_id: 0u32, - category_name: "Unknown".to_string(), + category_name: "Unknown".intern(), channels: vec![], }; @@ -165,19 +163,19 @@ pub async fn parse_xtream(input: &ConfigInput, let item_type = PlaylistItemType::from(xtream_cluster); let mut item = PlaylistItem { header: PlaylistItemHeader { - id: stream.get_stream_id().to_string(), + id: stream.get_stream_id().intern(), uuid: generate_playlist_uuid(&input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), - name: stream.get_name().to_string(), - logo: stream.get_stream_icon().to_string(), - group: interner.intern(category_name), - title: stream.get_name().to_string(), - url: stream_url.to_string(), - epg_channel_id: stream.get_epg_channel_id(), + name: Arc::clone(&stream.get_name()), + logo: Arc::clone(&stream.get_stream_icon()), + group: Arc::clone(category_name), + title: Arc::clone(&stream.get_name()), + url: stream_url.clone(), + epg_channel_id: stream.get_epg_channel_id().clone(), item_type, xtream_cluster, + category_id: stream.get_category_id(), additional_properties: Some(stream), - category_id: 0, - input_name: interner.intern(&input_name), + input_name: input_name.clone(), ..Default::default() }, }; @@ -197,7 +195,7 @@ pub async fn parse_xtream(input: &ConfigInput, PlaylistGroup { id: category.category_id, xtream_cluster, - title: interner.intern(&category.category_name), + title: Arc::clone(&category.category_name), channels: category.channels.clone(), } }).collect())) @@ -215,112 +213,115 @@ pub async fn parse_xtream_streaming( categories: DynReader, streams: DynReader, mut on_item: F, -) -> Result, TuliproxError> -where - F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> + Send + 'static +) -> Result, TuliproxError> +where + F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> + Send + 'static, { // 1. Parse Categories let xtream_categories = map_to_xtream_category(categories).await?; - + // 2. Prepare for Stream Parsing - let mut interner = StringInterner::new(); let input_name = input.name.clone(); - let url = input.url.as_str().to_string(); + let url = input.url.as_str().to_string(); let username = input.username.as_ref().map_or("", |v| v).to_string(); let password = input.password.as_ref().map_or("", |v| v).to_string(); let options = input.options.clone(); // Map categories for lookup - let group_map: IndexMap = xtream_categories.iter().map(|c| (c.category_id, c.category_name.clone())).collect(); - let unknown_group_name = "Unknown".to_string(); + let group_map: IndexMap> = xtream_categories.iter().map(|c| (c.category_id, c.category_name.clone())).collect(); + let unknown_group_name = "Unknown".intern(); spawn_blocking(move || { let reader = tokio_util::io::SyncIoBridge::new(streams); let mut deserializer = serde_json::Deserializer::from_reader(reader); - + let (live_stream_use_prefix, live_stream_without_extension) = options.as_ref() - .map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension)); + .map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension)); let mut source_ordinal = 0u32; match xtream_cluster { - XtreamCluster::Live => { - let mut on_stream = |stream: LiveStreamProperties| { - source_ordinal += 1; - let stream_prop = StreamProperties::Live(stream); - process_stream_item(&input_name, &url, &username, &password, - xtream_cluster, &mut interner, &group_map, &unknown_group_name, - stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) - }; - let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; - deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; - } - XtreamCluster::Video => { - let mut on_stream = |stream: VideoStreamProperties| { - source_ordinal += 1; - let stream_prop = StreamProperties::Video(Box::new(stream)); - process_stream_item(&input_name, &url, &username, &password, - xtream_cluster, &mut interner, &group_map, &unknown_group_name, - stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) - }; - let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; - deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; - } - XtreamCluster::Series => { - let mut on_stream = |stream: SeriesStreamProperties| { - source_ordinal += 1; - let stream_prop = StreamProperties::Series(Box::new(stream)); - process_stream_item(&input_name, &url, &username, &password, - xtream_cluster, &mut interner, &group_map, &unknown_group_name, - stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) - }; - let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; - deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; - } + XtreamCluster::Live => { + let mut on_stream = |stream: LiveStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Live(stream); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + XtreamCluster::Video => { + let mut on_stream = |stream: VideoStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Video(Box::new(stream)); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } + XtreamCluster::Series => { + let mut on_stream = |stream: SeriesStreamProperties| { + source_ordinal += 1; + let stream_prop = StreamProperties::Series(Box::new(stream)); + process_stream_item(&input_name, &url, &username, &password, + xtream_cluster, &group_map, &unknown_group_name, + stream_prop, &mut on_item, live_stream_use_prefix, live_stream_without_extension, source_ordinal) + }; + let visitor = XtreamItemVisitor { on_item: &mut on_stream, _marker: std::marker::PhantomData }; + deserializer.deserialize_any(visitor).map_err(|e| notify_err!("JSON parse error: {e}"))?; + } } Ok(()) }).await.map_err(|e| notify_err!("Streaming parse failed: {e}"))??; - Ok(xtream_categories) + Ok(xtream_categories) } #[allow(clippy::too_many_arguments)] fn process_stream_item( - input_name: &str, url: &str, username: &str, password: &str, - cluster: XtreamCluster, interner: &mut StringInterner, - group_map: &IndexMap, unknown_group_name: &str, + input_name: &Arc, + url: &str, username: &str, password: &str, + cluster: XtreamCluster, + group_map: &IndexMap>, + unknown_group_name: &Arc, mut stream: StreamProperties, callback: &mut F, - live_stream_use_prefix: bool, live_stream_without_extension: bool, + live_stream_use_prefix: bool, + live_stream_without_extension: bool, source_ordinal: u32, ) -> Result<(), TuliproxError> -where F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> +where + F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError>, { - stream.prepare(); - let category_id = stream.get_category_id(); - let category_name = group_map.get(&category_id).map_or(unknown_group_name, String::as_str); - let stream_url = create_xtream_url(cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); - - let item_type = PlaylistItemType::from(cluster); - let item = PlaylistItem { - header: PlaylistItemHeader { - id: stream.get_stream_id().to_string(), - uuid: generate_playlist_uuid(input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), - name: stream.get_name().to_string(), - logo: stream.get_stream_icon().to_string(), - group: interner.intern(category_name), - title: stream.get_name().to_string(), - url: stream_url.to_string(), - epg_channel_id: stream.get_epg_channel_id(), - item_type, - xtream_cluster: cluster, - additional_properties: Some(stream), - category_id, - source_ordinal, - input_name: interner.intern(input_name), - ..Default::default() - }, - }; + stream.prepare(); + let category_id = stream.get_category_id(); + let category_name = group_map.get(&category_id).unwrap_or(unknown_group_name); + let stream_url = create_xtream_url(cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); + + let item_type = PlaylistItemType::from(cluster); + let item = PlaylistItem { + header: PlaylistItemHeader { + id: stream.get_stream_id().intern(), + uuid: generate_playlist_uuid(input_name, &stream.get_stream_id().to_string(), item_type, &stream_url), + name: stream.get_name(), + logo: stream.get_stream_icon(), + group: category_name.clone(), + title: stream.get_name(), + url: stream_url, + epg_channel_id: stream.get_epg_channel_id(), + item_type, + xtream_cluster: cluster, + additional_properties: Some(stream), + category_id, + source_ordinal, + input_name: Arc::clone(input_name), + ..Default::default() + }, + }; // if let Some(StreamProperties::Series(props)) = item.header.additional_properties.as_mut() { // // We need to set category_id for Series properties just like parse_xtream might expect or use? @@ -329,7 +330,7 @@ where F: FnMut(XtreamPlaylistItem) -> Result<(), TuliproxError> // let _ = props; // } - callback(XtreamPlaylistItem::from(&item)) + callback(XtreamPlaylistItem::from(&item)) } struct XtreamItemVisitor<'a, T, F> { diff --git a/backend/src/processing/playlist_watch.rs b/backend/src/processing/playlist_watch.rs index dcd24e999..1a5f19e29 100644 --- a/backend/src/processing/playlist_watch.rs +++ b/backend/src/processing/playlist_watch.rs @@ -1,4 +1,5 @@ use std::collections::BTreeSet; +use std::sync::Arc; use std::path::{Path}; use log::{error, info}; use shared::model::{MsgKind, PlaylistGroup}; @@ -23,8 +24,8 @@ pub async fn process_group_watch(client: &reqwest::Client, cfg: &Config, target_ if let Ok(true) = tokio::fs::try_exists(&path).await { if let Some(loaded_tree) = load_watch_tree(&path).await { // Find elements in set2 but not in set1 - let added_difference: BTreeSet = new_tree.difference(&loaded_tree).cloned().collect(); - let removed_difference: BTreeSet = loaded_tree.difference(&new_tree).cloned().collect(); + let added_difference: BTreeSet> = new_tree.difference(&loaded_tree).cloned().collect(); + let removed_difference: BTreeSet> = loaded_tree.difference(&new_tree).cloned().collect(); if !added_difference.is_empty() || !removed_difference.is_empty() { changed = true; handle_watch_notification(client, cfg, &added_difference, &removed_difference, target_name, &pl.title).await; @@ -59,7 +60,7 @@ struct WatchChanges { pub removed: Vec, } -async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added: &BTreeSet, removed: &BTreeSet, target_name: &str, group_name: &str) { +async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added: &BTreeSet>, removed: &BTreeSet>, target_name: &str, group_name: &str) { let added = added.iter().map(std::string::ToString::to_string).collect::>(); let removed = removed.iter().map(std::string::ToString::to_string).collect::>(); if !added.is_empty() || !removed.is_empty() { @@ -77,12 +78,12 @@ async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added } } -async fn load_watch_tree(path: &Path) -> Option> { +async fn load_watch_tree(path: &Path) -> Option>> { let encoded = tokio::fs::read(path).await.ok()?; binary_deserialize(&encoded[..]).ok() } -async fn save_watch_tree(path: &Path, tree: &BTreeSet) -> std::io::Result<()> { +async fn save_watch_tree(path: &Path, tree: &BTreeSet>) -> std::io::Result<()> { let encoded: Vec = binary_serialize(&tree)?; tokio::fs::write(path, encoded).await } diff --git a/backend/src/processing/processor/epg.rs b/backend/src/processing/processor/epg.rs index 98895cf80..bcec64e30 100644 --- a/backend/src/processing/processor/epg.rs +++ b/backend/src/processing/processor/epg.rs @@ -169,9 +169,9 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> let mut icon_assigned = HashSet::new(); for epg_source in epg_sources { // icon tags - let icon_tags: HashMap<&String, &Arc> = epg_source.children.iter() + let icon_tags: HashMap<&str, &Arc> = epg_source.children.iter() .filter(|tag| tag.icon != XmlTagIcon::Undefined) - .filter_map(|tag| tag.get_attribute_value(EPG_ATTRIB_ID).map(|id| (id, tag))) + .filter_map(|tag| tag.get_attribute_value(EPG_ATTRIB_ID).map(|id| (id.as_str(), tag))) .collect(); let assign_values = |chan: &mut PlaylistItem| { @@ -180,7 +180,7 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> // if the channel has no epg_id or the epg_id is not present in xmltv/tvguide then we need to match one from existing tvguide let not_found_in_epg = match &chan.header.epg_channel_id { None => true, - Some(epg_id) => !id_cache.processed.contains(epg_id), + Some(epg_id) => !id_cache.processed.contains(&**epg_id), }; if not_found_in_epg { let try_match = |key: &str| { @@ -195,22 +195,22 @@ async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_> if let Some(new_id) = try_match(&chan.header.name) .or_else(|| chan.header.epg_channel_id.as_deref().and_then(try_match)) { - chan.header.epg_channel_id = Some(new_id); + chan.header.epg_channel_id = Some(new_id.into()); } } } if let Some(epg_channel_id) = chan.header.epg_channel_id.as_ref() { if !icon_assigned.contains(epg_channel_id) && (epg_source.logo_override || chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) { - if let Some(icon_tag) = icon_tags.get(epg_channel_id) { + if let Some(icon_tag) = icon_tags.get(&**epg_channel_id) { if let XmlTagIcon::Src(icon) = &icon_tag.icon { icon_assigned.insert(epg_channel_id.clone()); if epg_source.logo_override || chan.header.logo.is_empty() { trace!("Matched channel {} to epg icon {icon}", chan.header.name); - chan.header.logo = icon.clone(); + chan.header.logo = icon.clone().into(); } if epg_source.logo_override || chan.header.logo_small.is_empty() { - chan.header.logo_small = icon.clone(); + chan.header.logo_small = icon.clone().into(); } } } diff --git a/backend/src/processing/processor/library.rs b/backend/src/processing/processor/library.rs index 479c158f5..3d306a79a 100644 --- a/backend/src/processing/processor/library.rs +++ b/backend/src/processing/processor/library.rs @@ -1,12 +1,13 @@ -use std::collections::HashMap; use crate::library::{MediaMetadata, MetadataAsyncIter, MetadataCacheEntry}; use crate::model::{AppConfig, ConfigInput}; +use shared::concat_string; use shared::error::TuliproxError; -use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster}; -use shared::utils::{generate_playlist_uuid, StringInterner}; +use shared::model::UUIDType; +use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster}; +use shared::utils::{generate_playlist_uuid, Internable}; +use std::collections::HashMap; use std::path::Path; use std::sync::Arc; -use shared::concat_string; pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &Arc, input: &ConfigInput) -> (Vec, Vec) { let config = &*app_config.config.load(); @@ -27,14 +28,13 @@ pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &A channels: vec![], xtream_cluster: XtreamCluster::Series, }; - let mut interner = StringInterner::new(); while let Some(entry) = metadata_iter.next().await { match entry.metadata { MediaMetadata::Movie(_) => { - to_playlist_item(&mut interner, &entry, &input.name, &library_config.playlist.movie_category, &mut group_movies.channels); + to_playlist_item(&entry, &input.name, &library_config.playlist.movie_category, &mut group_movies.channels); } MediaMetadata::Series(_) => { - to_playlist_item(&mut interner, &entry, &input.name, &library_config.playlist.series_category, &mut group_series.channels); + to_playlist_item(&entry, &input.name, &library_config.playlist.series_category, &mut group_series.channels); } } } @@ -51,24 +51,26 @@ pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &A } -fn to_playlist_item(interner: &mut StringInterner, entry: &MetadataCacheEntry, input_name: &str, group_name: &str, channels: &mut Vec) { +fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &Arc, group_name: &str, channels: &mut Vec) { let metadata = &entry.metadata; match metadata { MediaMetadata::Movie(_) => { let additional_properties = metadata_cache_entry_to_xtream_movie_info(entry); + let title = metadata.title().intern(); + let group = group_name.intern(); channels.push(PlaylistItem { header: PlaylistItemHeader { uuid: UUIDType::from_valid_uuid(&entry.uuid), - name: metadata.title().to_string(), - group: interner.intern(group_name), - title: metadata.title().to_string(), - logo: metadata.poster().map_or_else(String::new, ToString::to_string), - url: format!("file://{}", entry.file_path), + name: Arc::clone(&title), + title, + group, + logo: metadata.poster().unwrap_or("").intern(), + url: concat_string!("file://", &entry.file_path).into(), xtream_cluster: XtreamCluster::Video, additional_properties, item_type: PlaylistItemType::LocalVideo, - input_name: interner.intern(input_name), + input_name: Arc::clone(input_name), ..PlaylistItemHeader::default() } }); @@ -76,30 +78,31 @@ fn to_playlist_item(interner: &mut StringInterner, entry: &MetadataCacheEntry, i MediaMetadata::Series(_series) => { if let Some(additional_properties) = metadata_cache_entry_to_xtream_series_info(entry) { let mut episodes = vec![]; + let group_arc: Arc = group_name.intern(); if let StreamProperties::Series(series_properties) = &additional_properties { if let Some(details_props) = series_properties.details.as_ref() { if let Some(prop_episodes) = details_props.episodes.as_ref() { for episode in prop_episodes { - let logo = if episode.movie_image.is_empty() { metadata.poster().map_or_else(String::new, ToString::to_string) } else { episode.movie_image.clone() }; - let container_extension = Path::new(&episode.direct_source) + let logo: Arc = if episode.movie_image.is_empty() { metadata.poster().unwrap_or("").intern() } else { episode.movie_image.clone() }; + let container_extension = Path::new(&*episode.direct_source) .extension() .and_then(|s| s.to_str()) .map(ToString::to_string).unwrap_or_default(); episodes.push(PlaylistItem { header: PlaylistItemHeader { - id: episode.id.to_string(), + id: episode.id.to_string().into(), // we use parent_code for local series to find the parent series info and straighten the virtual_ids - parent_code: entry.uuid.clone(), + parent_code: entry.uuid.clone().into(), uuid: generate_playlist_uuid(input_name, &episode.id.to_string(), PlaylistItemType::LocalSeries, &episode.direct_source), logo: logo.clone(), name: episode.title.clone(), - group: interner.intern(group_name), + group: Arc::clone(&group_arc), title: episode.title.clone(), url: episode.direct_source.clone(), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeries, category_id: 0, - input_name: interner.intern(input_name), + input_name: Arc::clone(input_name), additional_properties: Some(StreamProperties::Episode(EpisodeStreamProperties { episode_id: episode.id, episode: episode.episode_num, @@ -108,7 +111,7 @@ fn to_playlist_item(interner: &mut StringInterner, entry: &MetadataCacheEntry, i release_date: Some(episode.release_date.clone()), tmdb: episode.tmdb, movie_image: logo, - container_extension, + container_extension: container_extension.intern(), audio: None, video: None, })), @@ -123,15 +126,15 @@ fn to_playlist_item(interner: &mut StringInterner, entry: &MetadataCacheEntry, i let series_info = PlaylistItem { header: PlaylistItemHeader { uuid: UUIDType::from_valid_uuid(&entry.uuid), - id: entry.uuid.clone(), - name: metadata.title().to_string(), - group: interner.intern(group_name), - title: metadata.title().to_string(), - logo: metadata.poster().map_or_else(String::new, ToString::to_string), - url: format!("file://{}", entry.file_path), + id: entry.uuid.clone().into(), + name: metadata.title().intern(), + group: group_name.intern(), + title: metadata.title().intern(), + logo: metadata.poster().unwrap_or("").intern(), + url: concat_string!("file://", &entry.file_path).into(), xtream_cluster: XtreamCluster::Series, item_type: PlaylistItemType::LocalSeriesInfo, - input_name: interner.intern(input_name), + input_name: Arc::clone(input_name), additional_properties: Some(additional_properties), ..PlaylistItemHeader::default() } @@ -156,64 +159,64 @@ pub fn metadata_cache_entry_to_xtream_movie_info( .and_then(|s| s.to_str()) .map(ToString::to_string).unwrap_or_default(); - let actor_names = movie.actors.as_ref().map(|a| a.iter().map(|a| a.name.clone()).collect::>().join(", ")); + let actor_names = movie.actors.as_ref().map(|a| a.iter().map(|a| a.name.clone()).collect::>().join(", ").intern()); let properties = VideoStreamProperties { - name: movie.title.clone(), + name: movie.title.clone().into(), category_id: 0, stream_id: 0, - stream_icon: movie.poster.as_deref().or(movie.fanart.as_deref()).unwrap_or("").to_owned(), - direct_source: String::new(), + stream_icon: movie.poster.as_deref().or(movie.fanart.as_deref()).unwrap_or("").to_owned().into(), + direct_source: "".into(), custom_sid: None, - added: entry.file_modified.to_string(), - container_extension, + added: entry.file_modified.intern(), + container_extension: container_extension.intern(), rating: movie.rating, rating_5based: None, - stream_type: Some("movie".to_string()), - trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())), + stream_type: Some("movie".intern()), + trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone().into())), tmdb: movie.tmdb_id, is_adult: 0, details: Some(VideoStreamDetailProperties { - kinopoisk_url: movie.tmdb_id.map(|id| format!("https://www.themoviedb.org/movie/{id}")), - o_name: movie.original_title.clone(), - cover_big: movie.poster.clone(), - movie_image: movie.poster.clone(), - release_date: movie.year.map(|y| format!("{y}-01-01")), + kinopoisk_url: movie.tmdb_id.map(|id| concat_string!("https://www.themoviedb.org/movie/", &id.to_string()).into()), + o_name: movie.original_title.clone().map(Into::into), + cover_big: movie.poster.clone().map(Into::into), + movie_image: movie.poster.clone().map(Into::into), + release_date: movie.year.map(|y| format!("{y}-01-01").into()), episode_run_time: movie.runtime, - director: movie.directors.as_ref().map(|d| d.join(", ")), - youtube_trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())), + director: movie.directors.as_ref().map(|d| d.join(", ").into()), + youtube_trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone().into())), actors: actor_names.clone(), - cast: actor_names, - genre: movie.genres.as_ref().map(|g| g.join(", ")), - description: movie.plot.clone(), - plot: movie.plot.clone(), + cast: actor_names.clone(), + genre: movie.genres.as_ref().map(|g| g.join(", ").into()), + description: movie.plot.clone().map(Into::into), + plot: movie.plot.clone().map(Into::into), age: None, - mpaa_rating: movie.mpaa.clone(), + mpaa_rating: movie.mpaa.clone().map(Into::into), rating_count_kinopoisk: 0, country: None, backdrop_path: movie .fanart .as_ref() .filter(|s| !s.is_empty()) - .map(|f| vec![f.clone()]) + .map(|f| vec![f.clone().into()]) .or_else(|| { movie.poster .as_ref() .filter(|s| !s.is_empty()) - .map(|p| vec![p.clone()]) + .map(|p| vec![p.clone().into()]) }), - duration_secs: movie.runtime.map(|r| (r * 60).to_string()), + duration_secs: movie.runtime.map(|r| (r * 60).to_string().into()), duration: movie.runtime.map(|r| { let h = r / 60; let m = r % 60; - format!("{h:02}:{m:02}:00") + format!("{h:02}:{m:02}:00").into() }), video: None, audio: None, bitrate: 0, - runtime: movie.runtime.map(|r| (r * 60).to_string()), - status: Some("Released".to_string()), + runtime: movie.runtime.map(|r| (r * 60).to_string().into()), + status: Some("Released".intern()), }), }; @@ -229,22 +232,22 @@ pub fn metadata_cache_entry_to_xtream_series_info( MediaMetadata::Series(m) => m, }; - let actor_names = series.actors.as_ref().map(|a| a.iter().map(|a| a.name.clone()).collect::>().join(", ")).unwrap_or_default(); + let actor_names: Arc = series.actors.as_ref().map(|a| a.iter().map(|a| a.name.clone()).collect::>().join(", ")).unwrap_or_default().into(); let release_date = series.year.map(|y| format!("{y}-01-01")); let youtube_trailer = series.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())).unwrap_or_default(); let mut season_data = HashMap::new(); series.seasons.as_ref().iter().for_each(|seasons| seasons.iter().for_each(|season_metadata| { - season_data.insert(season_metadata.season_number,SeriesStreamDetailSeasonProperties { - name: season_metadata.name.clone(), + season_data.insert(season_metadata.season_number, SeriesStreamDetailSeasonProperties { + name: season_metadata.name.clone().into(), season_number: season_metadata.season_number, episode_count: 0, - overview: season_metadata.overview.clone(), - air_date: season_metadata.air_date.clone(), - cover: season_metadata.poster_path.clone(), - cover_tmdb: season_metadata.poster_path.clone(), + overview: season_metadata.overview.clone().map(Into::into), + air_date: season_metadata.air_date.clone().map(Into::into), + cover: season_metadata.poster_path.clone().map(Into::into), + cover_tmdb: season_metadata.poster_path.clone().map(Into::into), cover_big: None, - duration: Some(String::from("0")), + duration: Some(String::from("0").into()), }); })); @@ -258,39 +261,39 @@ pub fn metadata_cache_entry_to_xtream_series_info( let episode_release_date = episode.aired.as_ref().map(ToString::to_string).unwrap_or_default(); let tmdb_id = (episode.tmdb_id > 0).then_some(episode.tmdb_id); - let season_entry =season_data.entry(episode.season).or_insert_with(|| { + let season_entry = season_data.entry(episode.season).or_insert_with(|| { SeriesStreamDetailSeasonProperties { - name: concat_string!(&series.title, " ", &episode.season.to_string()), + name: concat_string!(&series.title, " ", &episode.season.to_string()).into(), season_number: episode.season, episode_count: 0, - overview: series.poster.clone(), - air_date: episode.aired.clone(), - cover: series.poster.clone(), + overview: series.poster.clone().map(Into::into), + air_date: episode.aired.clone().map(Into::into), + cover: series.poster.clone().map(Into::into), cover_tmdb: None, cover_big: None, duration: None, } - }); - season_entry.episode_count = season_entry.episode_count.saturating_add(1); + }); + season_entry.episode_count = season_entry.episode_count.saturating_add(1); SeriesStreamDetailEpisodeProperties { id: tmdb_id.unwrap_or_default(), episode_num: episode.episode, season: episode.season, - title: episode.title.clone(), - container_extension, + title: episode.title.clone().into(), + container_extension: container_extension.into(), custom_sid: None, - added: episode.file_modified.to_string(), - direct_source: episode.file_path.clone(), + added: episode.file_modified.to_string().into(), + direct_source: episode.file_path.clone().into(), tmdb: tmdb_id, - release_date: episode_release_date.clone(), - plot: episode.plot.clone(), - crew: Some(actor_names.clone()), + release_date: episode_release_date.clone().into(), + plot: episode.plot.clone().map(Into::into), + crew: Some(Arc::clone(&actor_names)), duration_secs: episode.runtime.map_or(0, |r| r * 60), duration: episode.runtime .map(|r| format!("{:02}:{:02}:00", r / 60, r % 60)) - .unwrap_or_default(), - movie_image: episode.thumb.clone().unwrap_or_default(), + .unwrap_or_default().into(), + movie_image: episode.thumb.clone().unwrap_or_default().into(), audio: None, video: None, bitrate: 0, @@ -304,30 +307,30 @@ pub fn metadata_cache_entry_to_xtream_series_info( seasons.sort_by_key(|s| s.season_number); let properties = SeriesStreamProperties { - name: series.title.clone(), + name: series.title.clone().into(), category_id: 0, series_id: 0, backdrop_path: series .fanart .as_ref() .filter(|s| !s.is_empty()) - .map(|f| vec![f.clone()]) + .map(|f| vec![f.clone().into()]) .or_else(|| { series.poster.as_ref() .filter(|s| !s.is_empty()) - .map(|p| vec![p.clone()]) + .map(|p| vec![p.clone().into()]) }), - cast: actor_names, - cover: series.poster.clone().unwrap_or_default(), - director: series.directors.as_ref().map(|d| d.join(", ")).unwrap_or_default(), + cast: Arc::clone(&actor_names), + cover: series.poster.clone().unwrap_or_default().into(), + director: series.directors.as_ref().map(|d| d.join(", ")).unwrap_or_default().into(), episode_run_time: None, - genre: series.genres.as_ref().map(|d| d.join(", ")), - last_modified: Some(series.last_updated.to_string()), - plot: series.plot.clone(), + genre: series.genres.as_ref().map(|d| d.join(", ").into()), + last_modified: Some(series.last_updated.to_string().into()), + plot: series.plot.clone().map(Into::into), rating: series.rating.unwrap_or(0f64), rating_5based: 0.0, - release_date, - youtube_trailer, + release_date: release_date.map(Into::into), + youtube_trailer: youtube_trailer.into(), tmdb: series.tmdb_id, details: Some(SeriesStreamDetailProperties { year: series.year, diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 8a1070def..4176d7d4c 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -40,9 +40,11 @@ use shared::foundation::filter::{get_field_value, set_field_value, Filter, Value use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState, - ProcessingOrder, UUIDType, XtreamCluster}; -use shared::utils::{create_alias_uuid, default_as_default, StringInterner}; + ProcessingOrder, XtreamCluster}; +use shared::utils::{create_alias_uuid, default_as_default, interner_gc, Internable}; use std::time::Instant; +use shared::concat_string; +use shared::model::UUIDType; fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { let provider = ValueProvider { pli, match_as_ascii }; @@ -71,7 +73,7 @@ pub fn apply_filter_to_source(source: &mut dyn PlaylistSource, filter: &Filter) if groups.is_empty() { None } else { Some(groups.into_values().collect()) } } -fn filter_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, _interner: &mut StringInterner) -> Option> { +fn filter_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Option> { apply_filter_to_source(source, &target.filter) } @@ -111,24 +113,24 @@ fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { } } -fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>, interner: &mut StringInterner) { +fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>) { if let Some(renames) = rename { if !renames.is_empty() { let result = pli; for r in renames { let value = get_field_value(result, r.field); let cap = r.pattern.replace_all(&value, &r.new_name); - if log_enabled!(log::Level::Debug) && value != cap { + if log_enabled!(log::Level::Debug) && *value != *cap { trace_if_enabled!("Renamed {}={value} to {cap}", &r.field); } let value = cap.into_owned(); - set_field_value(result, r.field, value, interner); + set_field_value(result, r.field, value); } } } } -fn rename_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, interner: &mut StringInterner) -> Option> { +fn rename_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Option> { match &target.rename { Some(renames) if !renames.is_empty() => { let mut groups: IndexMap<(XtreamCluster, Arc), PlaylistGroup> = IndexMap::new(); @@ -139,11 +141,11 @@ fn rename_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, inter let value = &*pli.header.group; let cap = r.pattern.replace_all(value, &r.new_name); if *value != cap { - pli.header.group = interner.intern(&cap); + pli.header.group = cap.intern(); } } } - exec_rename(&mut pli, Some(renames), interner); + exec_rename(&mut pli, Some(renames)); let group_title = pli.header.group.clone(); let cluster = pli.header.xtream_cluster; let cat_id = pli.header.category_id; @@ -197,7 +199,7 @@ fn map_channel_and_flatten(channel: PlaylistItem, mapping: &Mapping) -> Vec Option> { +fn map_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget) -> Option> { let mapping_binding = target.mapping.load(); let mappings = mapping_binding.as_ref()?; let valid_mappings = mappings.iter().filter(|m| m.mapper.as_ref().is_some_and(|v| !v.is_empty())); @@ -368,10 +370,9 @@ async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> (Vec, Vec, Vec) { let sources = ctx.config.sources.load(); let mut errors = vec![]; - let mut input_stats = HashMap::::new(); + let mut input_stats = HashMap::, InputStats>::new(); let mut target_stats = Vec::::new(); if let Some(source) = sources.get_source_at(source_idx) { - let mut interner = StringInterner::default(); let mut source_playlists = Vec::with_capacity(128); let broadcast_step = create_broadcast_callback(ctx.event_manager.as_ref()); // Download the sources @@ -402,7 +403,7 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> ( let tvguide = if input.input_type == InputType::Library { None } else { - download_input_epg(ctx,input, &mut error_list).await + download_input_epg(ctx, input, &mut error_list).await }; errors.append(&mut error_list); @@ -430,14 +431,13 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> ( if source_downloaded { if source_playlists.is_empty() { debug!("Source at index {source_idx} is empty"); - errors.push(notify_err!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(std::string::String::as_str).collect::>().join(", "))); + errors.push(notify_err!("Source at index {source_idx} is empty: {}", source.inputs.iter().map(Clone::clone).collect::>>().join(", "))); } else { debug_if_enabled!("Source has {} groups", source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::()); for target in &source.targets { if is_target_enabled(target, &ctx.user_targets) { match process_playlist_for_target(ctx, &mut source_playlists, target, - &mut input_stats, &mut errors, - &mut interner).await { + &mut input_stats, &mut errors).await { Ok(()) => { target_stats.push(TargetStats::success(&target.name)); } @@ -454,7 +454,7 @@ async fn process_source(source_idx: usize, ctx: &PlaylistProcessingContext) -> ( (input_stats.into_values().collect(), target_stats, errors) } -async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc, +async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc, error_list: &mut Vec) -> Option { // Download epg for input let (tvguide, mut tvguide_errors) = if error_list.is_empty() { @@ -468,7 +468,7 @@ async fn download_input_epg(ctx: &PlaylistProcessingContext, input: &Arc) - -> (Vec, Box, Option) { + -> (Vec, Box, Option) { // Coordination Logic let need_download = !ctx.is_input_downloaded(&input.name).await; @@ -544,8 +544,9 @@ pub struct PlaylistProcessingContext { pub playlist_state: Option>, // Coordination - processed_inputs: Arc>>, - input_locks: Arc>>>>, + processed_inputs: Arc>>>, + #[allow(clippy::type_complexity)] + input_locks: Arc, Weak>>>>, } impl PlaylistProcessingContext { @@ -553,19 +554,19 @@ impl PlaylistProcessingContext { let processed = self.processed_inputs.lock().await; processed.contains(input_name) } - pub async fn mark_input_downloaded(&self, input_name: String) -> bool { + pub async fn mark_input_downloaded(&self, input_name: Arc) -> bool { let mut processed = self.processed_inputs.lock().await; processed.insert(input_name) } - pub async fn get_input_lock(&self, input_name: &str) -> OwnedRwLockWriteGuard<()> { + pub async fn get_input_lock(&self, input_name: &Arc) -> OwnedRwLockWriteGuard<()> { let mut locks = self.input_locks.lock().await; // Try to upgrade the existing weak reference let lock = locks.get(input_name) .and_then(Weak::upgrade) .unwrap_or_else(|| { let new_lock = Arc::new(RwLock::new(())); - locks.insert(input_name.to_string(), Arc::downgrade(&new_lock)); + locks.insert(input_name.clone(), Arc::downgrade(&new_lock)); new_lock }); @@ -594,7 +595,7 @@ async fn process_sources(processing_ctx: &PlaylistProcessingContext) -> (Vec (Vec Option>>; +pub type ProcessingPipe = Vec Option>>; fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { match &target.processing_order { @@ -652,8 +653,7 @@ fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { } fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>, - duplicates: &mut HashSet, - interner: &mut StringInterner) -> FetchedPlaylist<'a> { + duplicates: &mut HashSet) -> FetchedPlaylist<'a> { let mut new_fpl = FetchedPlaylist { input: fpl.input, source: fpl.clone_source(), @@ -664,7 +664,7 @@ fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedP } for f in pipe { - if let Some(groups) = f(new_fpl.source.as_mut(), target, interner) { + if let Some(groups) = f(new_fpl.source.as_mut(), target) { new_fpl.source = MemoryPlaylistSource::new(groups).boxed(); } } @@ -703,9 +703,8 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { async fn process_playlist_for_target(ctx: &PlaylistProcessingContext, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, - stats: &mut HashMap, + stats: &mut HashMap, InputStats>, errors: &mut Vec, - interner: &mut StringInterner, ) -> Result<(), Vec> { debug_if_enabled!("Processing order is {}", &target.processing_order); @@ -719,9 +718,9 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext, let mut step = StepMeasure::new(&target.name, broadcast_step); for provider_fpl in playlists.iter_mut() { step.broadcast("Executing transformations on '{}' playlist", &target.name); - let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates, interner); + let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); processed_fpl.sort_by_provider_ordinal(); - playlist_resolve_series(&ctx.config, &ctx.client, target, errors, &pipe, provider_fpl, &mut processed_fpl, interner).await; + playlist_resolve_series(&ctx.config, &ctx.client, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; playlist_resolve_vod(&ctx.config, &ctx.client, target, errors, provider_fpl, &mut processed_fpl).await; // stats let input_entry_name = processed_fpl.input.name.clone(); @@ -767,7 +766,7 @@ async fn process_playlist_for_target(ctx: &PlaylistProcessingContext, if process_watch(&config, &ctx.client, target, &flat_new_playlist).await { step.tick("group watches"); } - let result = persist_playlist(&ctx.config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, ctx.playlist_state.as_ref(), interner).await; + let result = persist_playlist(&ctx.config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, ctx.playlist_state.as_ref()).await; step.stop("Persisting playlists"); result } @@ -866,7 +865,7 @@ async fn process_watch(cfg: &Config, client: &reqwest::Client, target: &ConfigTa pub async fn exec_processing(client: &reqwest::Client, app_config: Arc, targets: Arc, event_manager: Option>, playlist_state: Option>, update_guard: Option) { - let _guard = if let Some(guard) = update_guard { + let _guard = if let Some(guard) = update_guard { if let Some(permit) = guard.try_playlist() { Some(permit) } else { @@ -936,6 +935,8 @@ pub async fn exec_processing(client: &reqwest::Client, app_config: Arc>>(); let sorted = channels .into_iter() @@ -340,7 +341,7 @@ mod tests { .enumerate() .map(|(i, name)| PlaylistItem { header: PlaylistItemHeader { - title: name.to_string(), + title: name.to_string().into(), source_ordinal: i as u32, ..Default::default() }, @@ -404,7 +405,7 @@ mod tests { "US| East d", "US| West e", "US| West f", - ]; + ].into_iter().map(Into::into).collect::>>(); let sorted = channels .into_iter() diff --git a/backend/src/processing/processor/trakt.rs b/backend/src/processing/processor/trakt.rs index 20db6845d..a7ead6731 100644 --- a/backend/src/processing/processor/trakt.rs +++ b/backend/src/processing/processor/trakt.rs @@ -4,11 +4,10 @@ use crate::utils::{extract_year_from_title, normalize_title_for_matching, TraktC use crate::utils::{trace_if_enabled, with}; use log::{debug, info, trace, warn}; use shared::error::TuliproxError; -use shared::utils::StringInterner; use shared::model::{FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, TraktContentType, XtreamCluster}; -use shared::utils::CONSTANTS; +use shared::utils::{Internable, CONSTANTS}; use indexmap::IndexMap; -use std::borrow::Cow; +use std::sync::Arc; use strsim::normalized_levenshtein; fn extract_quality(value: &str) -> Option<&str> { @@ -130,7 +129,6 @@ fn find_best_match_for_item<'a>( fn create_category_from_matches<'a>( matches: Vec>, list_config: &'a TraktListConfig, - interner: &mut StringInterner, ) -> Vec { if matches.is_empty() { return vec![]; } @@ -147,12 +145,12 @@ fn create_category_from_matches<'a>( )) }); - let group_title = &list_config.category_name; + let group_title = list_config.category_name.as_str().intern(); for match_result in sorted_matches { let mut modified_item = match_result.playlist_item.clone(); with!(mut modified_item.header => header { - let title = header.get_field("caption").unwrap_or_else(|| Cow::Borrowed(&header.title)); + let title = header.get_field("caption").unwrap_or_else(|| Arc::clone(&header.title)); if extract_quality(&title).is_none() { if let Some(quality) = extract_quality(&header.group) { let mut caption = String::with_capacity(title.len() + 6); @@ -163,7 +161,7 @@ fn create_category_from_matches<'a>( header.set_field("caption", &caption); } } - header.group = interner.intern(group_title); + header.group = group_title.clone(); header.gen_uuid(); matched_items_by_cluster.entry(header.xtream_cluster).or_default().push(modified_item); }); @@ -172,7 +170,7 @@ fn create_category_from_matches<'a>( matched_items_by_cluster.into_iter().map(|(cluster, channels)| { PlaylistGroup { id: 0, - title: interner.intern(group_title), + title: group_title.clone(), channels, xtream_cluster: cluster, } @@ -183,7 +181,6 @@ fn match_trakt_items_with_playlist<'a>( trakt_items: &'a [TraktListItem], playlist: &'a [PlaylistGroup], list_config: &'a TraktListConfig, - interner: &mut StringInterner, ) -> Vec { let trakt_match_items: Vec> = trakt_items .iter() @@ -207,7 +204,7 @@ fn match_trakt_items_with_playlist<'a>( } } - create_category_from_matches(matches, list_config, interner) + create_category_from_matches(matches, list_config) } pub struct TraktCategoriesProcessor { @@ -234,7 +231,6 @@ impl TraktCategoriesProcessor { info!("Processing {} Trakt lists for target {}", trakt_config.lists.len(), target.name); let mut new_categories = Vec::new(); let mut total_matches = 0; - let mut interner = StringInterner::new(); for list_config in &trakt_config.lists { let cache_key = format!("{}:{}", list_config.user, list_config.list_slug); @@ -243,7 +239,7 @@ impl TraktCategoriesProcessor { Ok(trakt_items) => { debug!("Processing Trakt list {cache_key} with {} items", trakt_items.len()); - let categories = match_trakt_items_with_playlist(&trakt_items, playlist, list_config, &mut interner); + let categories = match_trakt_items_with_playlist(&trakt_items, playlist, list_config); for category in categories { if !category.channels.is_empty() { total_matches += category.channels.len(); @@ -275,6 +271,9 @@ pub async fn process_trakt_categories_for_target( debug!("No Trakt configuration found for target {}", target.name); return Ok(None); }; + if !trakt_config.enabled { + return Ok(None); + } let processor = TraktCategoriesProcessor::new(http_client, trakt_config); processor.process_trakt_categories(playlist, target, trakt_config).await diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 3a10d34f1..90229d6d9 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -11,7 +11,6 @@ use log::{error, info, log_enabled, Level}; use shared::error::TuliproxError; use shared::model::{InputType, PlaylistEntry, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo}; use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster}; -use shared::utils::StringInterner; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; @@ -25,23 +24,27 @@ async fn playlist_resolve_series_info(app_config: &Arc, client: &reqw errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_series: bool, - resolve_delay: u16, - interner: &mut StringInterner) -> Vec { + resolve_delay: u16) -> Vec { + let input = fpl.input; let working_dir = &app_config.config.load().working_dir; let storage_path = match get_input_storage_path(&input.name, working_dir) { Ok(storage_path) => storage_path, Err(err) => { - error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name); + error!("Can't resolve series info, input storage directory for input '{}' failed: {err}", input.name); return vec![]; } }; - let series_info_count = fpl.get_missing_series_info_count(); - - if series_info_count > 0 { - info!("Found {series_info_count} series info to resolve"); - } + let series_info_count = if resolve_series { + let series_info_count = fpl.get_missing_series_info_count(); + if series_info_count > 0 { + info!("Found {series_info_count} series info to resolve"); + } + series_info_count + } else { + 0 + }; let mut last_log_time = Instant::now(); let mut processed_series_info_count = 0; @@ -94,7 +97,7 @@ async fn playlist_resolve_series_info(app_config: &Arc, client: &reqw let header = &pli.header; (header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() }) }; - if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input, interner) { + if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), properties, &group, &series_name, input) { let group = group_series.entry(pli.header.category_id) .or_insert_with(|| { PlaylistGroup { @@ -134,12 +137,11 @@ pub async fn playlist_resolve_series(cfg: &Arc, pipe: &ProcessingPipe, provider_fpl: &mut FetchedPlaylist<'_>, processed_fpl: &mut FetchedPlaylist<'_>, - interner: &mut StringInterner, ) { let (resolve_series, resolve_delay) = get_resolve_series_options(target, processed_fpl); provider_fpl.source.release_resources(XtreamCluster::Series); - let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay, interner).await; + let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_series, resolve_delay).await; provider_fpl.source.obtain_resources().await; if series_playlist.is_empty() { return; } @@ -153,7 +155,7 @@ pub async fn playlist_resolve_series(cfg: &Arc, let mut new_playlist = series_playlist; for f in pipe { let mut source = MemoryPlaylistSource::new(new_playlist); - if let Some(v) = f(&mut source, target, interner) { + if let Some(v) = f(&mut source, target) { new_playlist = v; } else { new_playlist = source.take_groups(); diff --git a/backend/src/repository/bplustree.rs b/backend/src/repository/bplustree.rs index cf01b39c9..ac7eb7ee8 100644 --- a/backend/src/repository/bplustree.rs +++ b/backend/src/repository/bplustree.rs @@ -1,8 +1,10 @@ +use crate::repository::storage::get_file_path_for_db_index; use crate::utils; use crate::utils::{binary_deserialize, binary_serialize}; -use fs2::FileExt; +use fs2::FileExt as _; use indexmap::IndexMap; use log::error; +use memmap2::Mmap; use serde::{Deserialize, Serialize}; use shared::error::{string_to_io_error, to_io_error}; use std::ffi::OsString; @@ -10,9 +12,10 @@ use std::fs::{File, OpenOptions}; use std::io::{self, BufReader, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; use std::mem::size_of; +#[cfg(unix)] +use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use tempfile::NamedTempFile; -use crate::repository::storage::get_file_path_for_db_index; // Constants (Restored) const PAGE_SIZE: u16 = 4096; @@ -23,6 +26,9 @@ const MAGIC: &[u8; 4] = b"BTRE"; const STORAGE_VERSION: u32 = 1; const HEADER_SIZE: u64 = PAGE_SIZE as u64; const ROOT_OFFSET_POS: u64 = 8; +const METADATA_OFFSET_POS: u64 = 16; +// Reserve space for metadata (e.g. 4096 - 16 = 4080 bytes max, but let's be safe) +const METADATA_MAX_SIZE: usize = 4000; const POINTER_SIZE: usize = 8; const INFO_SIZE: usize = 12; // (u64, u32) @@ -49,6 +55,8 @@ const PAGE_HEADER_SIZE: u16 = 16; const PAGE_HEADER_SIZE_USIZE: usize = PAGE_HEADER_SIZE as usize; const SLOT_SIZE: usize = 2; // u16 +const MAGIC_METADATA_TARGET_ID_MAPPING: u8 = 0x01; + /* Page Header Layout @@ -1140,7 +1148,8 @@ where file.read_exact(&mut buffer[0..header_required])?; let is_leaf = buffer[0] != 0; - let keys_len = u32_from_bytes(&buffer[FLAG_SIZE..=LEN_SIZE])? as usize; + #[allow(clippy::range_plus_one)] + let keys_len = u32_from_bytes(&buffer[FLAG_SIZE..FLAG_SIZE + LEN_SIZE])? as usize; let min_required = header_required + keys_len + LEN_SIZE; if buffer.len() < min_required { @@ -1149,7 +1158,7 @@ where file.read_exact(&mut buffer[header_required..min_required])?; - let mut read_pos = FLAG_SIZE + LEN_SIZE; + let mut read_pos = header_required; let keys: Vec = binary_deserialize(&buffer[read_pos..read_pos + keys_len])?; read_pos += keys_len; @@ -1194,6 +1203,47 @@ where Ok((Self { keys, children, is_leaf, value_info, values }, children_pointer)) } + fn deserialize_from_mmap( + mmap: &[u8], + file: &mut R, + offset: u64, + nested: bool, + ) -> io::Result<(Self, Option>)> { + let start = usize::try_from(offset).map_err(to_io_error)?; + let header_end = start.checked_add(FLAG_SIZE + LEN_SIZE) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap offset overflow"))?; + // Basic safety check for mmap bounds + if header_end > mmap.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap access out of bounds")); + } + + let keys_len = u32_from_bytes(&mmap[start + FLAG_SIZE..start + FLAG_SIZE + LEN_SIZE])? as usize; + let keys_start = header_end; + let len_pos = keys_start + .checked_add(keys_len) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap offset overflow"))?; + + if len_pos + LEN_SIZE > mmap.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap access out of bounds")); + } + let payload_len = u32_from_bytes(&mmap[len_pos..len_pos + LEN_SIZE])? as usize; + let total = len_pos + .checked_add(LEN_SIZE + payload_len) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap offset overflow"))?; + + if total > mmap.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap access out of bounds")); + } + + // We need to know the total size of the node to slice the mmap + // For simplicity, we can just slice a PAGE_SIZE or slightly more if we know it overflows. + // Actually, our serialize_to_block uses PAGE_SIZE blocks. + + //let slice = &mmap[start..]; + let slice = &mmap[start..total]; + Self::deserialize_from_block_slice(slice, file, nested) + } + fn deserialize_from_block_slice( slice: &[u8], file: &mut R, @@ -1345,12 +1395,44 @@ where binary_deserialize(&data) } } +// ----------------------------------------------------------------------------- +// Metadata Enum +// ----------------------------------------------------------------------------- +#[derive(Clone, Debug, PartialEq)] +pub enum BPlusTreeMetadata { + Empty, + TargetIdMapping(u32), +} + +impl BPlusTreeMetadata { + pub fn to_bytes(&self) -> Vec { + match self { + Self::Empty => Vec::new(), + Self::TargetIdMapping(val) => { + let mut bytes = vec![MAGIC_METADATA_TARGET_ID_MAPPING]; // Type tag + bytes.extend_from_slice(&val.to_le_bytes()); + bytes + } + } + } + + pub fn from_bytes(bytes: &[u8]) -> Self { + match bytes.len() { + 5 if bytes[0] == MAGIC_METADATA_TARGET_ID_MAPPING => { + let arr: [u8; 4] = bytes[1..5].try_into().unwrap_or([0; 4]); + Self::TargetIdMapping(u32::from_le_bytes(arr)) + } + _ => Self::Empty, // Unknown metadata treated as Empty for now + } + } +} #[derive(Debug, Clone)] pub struct BPlusTree { root: BPlusTreeNode, inner_order: usize, leaf_order: usize, + metadata: BPlusTreeMetadata, dirty: bool, } @@ -1391,10 +1473,22 @@ where root: BPlusTreeNode::::new(true), inner_order, leaf_order, + metadata: BPlusTreeMetadata::Empty, dirty: true, // an empty tree is stored! } } + /// Helper to access metadata + pub fn get_metadata(&self) -> &BPlusTreeMetadata { + &self.metadata + } + + /// Helper to set metadata + pub fn set_metadata(&mut self, data: BPlusTreeMetadata) { + self.metadata = data; + self.dirty = true; + } + pub fn is_empty(&self) -> bool { self.root.keys.is_empty() } @@ -1476,7 +1570,7 @@ where pub fn store_index(filepath: &Path, sort_key_extractor: F) -> io::Result<()> where SortKey: Ord + Serialize, - F: Fn(&V) -> SortKey + F: Fn(&V) -> SortKey, { let index_path = get_file_path_for_db_index(filepath); @@ -1515,8 +1609,22 @@ where header[4..8].copy_from_slice(&STORAGE_VERSION.to_le_bytes()); // Placeholder for root offset, will be updated after serialization header[8..16].copy_from_slice(&HEADER_SIZE.to_le_bytes()); + + let meta_bytes = self.metadata.to_bytes(); + if meta_bytes.len() > METADATA_MAX_SIZE || 20 + meta_bytes.len() > PAGE_SIZE_USIZE { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Metadata too large for header page")); + } + if !meta_bytes.is_empty() { + let len = u32::try_from(meta_bytes.len()).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + header[16..20].copy_from_slice(&len.to_le_bytes()); + header[20..20 + meta_bytes.len()].copy_from_slice(&meta_bytes); + } + file.write_all(&header)?; + // We need to ensure we pad to PAGE_SIZE before continuing + file.seek(SeekFrom::Start(HEADER_SIZE))?; + // Use breadth-first serialization for better disk locality match self.root.serialize_breadth_first(&mut file, &mut buffer, HEADER_SIZE) { Ok(result) => { @@ -1535,79 +1643,37 @@ where /// Bulk build a tree from pre-calculated `ValueInfos` (streaming compact helper). /// Writes nodes to `file` starting at `start_offset`. /// Returns the offset of the root node used to update the file header. - fn bulk_build_from_infos( - &mut self, + fn build_levels_from_pointers( + &self, file: &mut W, - items: &[(K, ValueInfo)], - start_offset: u64, + mut next_level_pointers: Vec<(K, u64)>, + mut current_offset: u64, + write_buffer: &mut [u8], ) -> io::Result { - if items.is_empty() { - // Write empty root - let root = BPlusTreeNode::::new(true); - let mut buffer = Vec::with_capacity(PAGE_SIZE_USIZE); - let _offset = root.serialize_to_block(file, &mut buffer, start_offset)?; - // For a single empty root, the root offset is the start offset. - // serialize_to_block returns the *end* offset (start of next block). - // But header needs the *start* of the root node. - // Wait - serialize_to_block logic: - // "Returns the offset of the NEXT block" - // So the root IS at start_offset. - return Ok(start_offset); + if next_level_pointers.is_empty() { + return Ok(current_offset); } - // 1. Sort items (Query iterator is usually sorted, but let's be safe or rely on caller) - // Since BPlusTreeQuery works on offsets, order might be physical not logical if we iterate blocks? - // BPlusTreeQuery::iter() uses range_scan which traverses logical order. So it IS sorted! - - let mut current_offset = start_offset; - let mut write_buffer = Vec::with_capacity(PAGE_SIZE_USIZE); - - // 2. Build Leaves - // Chunk items into leaf nodes - let mut next_level_pointers: Vec<(K, u64)> = Vec::new(); - - for chunk in items.chunks(self.leaf_order) { - let mut node = BPlusTreeNode::::new(true); - for (k, v) in chunk { - node.keys.push(k.clone()); - node.value_info.push(v.clone()); - } - - let node_offset = current_offset; - current_offset = node.serialize_to_block(file, &mut write_buffer, node_offset)?; - - if !node.keys.is_empty() { - next_level_pointers.push((node.keys[0].clone(), node_offset)); - } - } - - // 3. Build Internal Levels recursively while next_level_pointers.len() > 1 { let mut parent_level_pointers: Vec<(K, u64)> = Vec::new(); - let children = next_level_pointers; // Move + let children = next_level_pointers; - // Chunk size: `inner_order + 1` (pointers). for chunk in children.chunks(self.inner_order + 1) { let mut node = BPlusTreeNode::::new(false); let mut pointers = Vec::new(); - // First pointer if let Some((_, off)) = chunk.first() { pointers.push(*off); } - // Subsequent pointers and separators for (k, off) in &chunk[1..] { node.keys.push(k.clone()); pointers.push(*off); } let node_offset = current_offset; + current_offset = node.serialize_internal_with_offsets(file, write_buffer, node_offset, &pointers)?; - // Use serialize_internal_with_offsets - current_offset = node.serialize_internal_with_offsets(file, &mut write_buffer, node_offset, &pointers)?; - - // Track for upper level if let Some((k, _)) = chunk.first() { parent_level_pointers.push((k.clone(), node_offset)); } @@ -1615,19 +1681,19 @@ where next_level_pointers = parent_level_pointers; } - // 4. Return Root Offset if let Some((_, root_off)) = next_level_pointers.first() { Ok(*root_off) } else { - Ok(start_offset) + Ok(current_offset) } } + pub fn load(filepath: &Path) -> io::Result { let mut file = File::open(filepath)?; // Verify Header - let mut header = [0u8; 16]; + let mut header = [0u8; PAGE_SIZE_USIZE]; // Read the whole header page file.read_exact(&mut header)?; if &header[0..4] != MAGIC { return Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid magic number")); @@ -1638,6 +1704,17 @@ where } let root_offset = u64::from_le_bytes(header[8..16].try_into().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid root offset slice"))?); + // Read metadata + let metadata_len = u32::from_le_bytes(header[16..20].try_into().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid metadata length slice"))?); + let metadata = if metadata_len > 0 { + if (20 + metadata_len as usize) > PAGE_SIZE_USIZE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Metadata length exceeds header page size")); + } + header[20..(20 + metadata_len as usize)].to_vec() + } else { + Vec::new() + }; + let mut reader = utils::file_reader(file); let mut buffer = vec![0u8; PAGE_SIZE_USIZE]; // Start after header block @@ -1648,6 +1725,7 @@ where root, inner_order, leaf_order, + metadata: BPlusTreeMetadata::from_bytes(&metadata), dirty: false, }) } @@ -1683,25 +1761,18 @@ where let mut offset = start_offset; loop { // Try Cache First - let (node, pointers) = if let Some(cached_block) = cache.get_mut(&offset) { - // Move to end (LRU) - let data = cached_block.clone(); - // We use a fresh buffer for the deserializer to avoid borrowing conflicts - let temp_buffer = data; - let res = BPlusTreeNode::::deserialize_from_block_slice(&temp_buffer, file, false) + let (node, pointers) = if let Some(data) = cache.shift_remove(&offset) { + // LRU: Use shift_remove + insert to avoid cloning or repeated lookups + let res = BPlusTreeNode::::deserialize_from_block_slice(&data, file, false) .map_err(BPlusTreeError::from)?; - - // Re-insert to mark as MRU - if let Some(removed) = cache.shift_remove(&offset) { - cache.insert(offset, removed); - } + cache.insert(offset, data); res } else { // Disk Read match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { Ok((node, pointers)) => { // Update Cache - if cache.len() >= CACHE_CAPACITY { // Cap at ~4MB of blocks + if cache.len() >= CACHE_CAPACITY { cache.shift_remove_index(0); } cache.insert(offset, buffer.to_owned()); @@ -1755,17 +1826,18 @@ where { let mut offset = start_offset; loop { - let (node, pointers) = if let Some(cached_block) = cache.get_mut(&offset) { - let temp_buffer = cached_block.clone(); - let res = BPlusTreeNode::::deserialize_from_block_slice(&temp_buffer, file, false) + // Try Cache First + let (node, pointers) = if let Some(data) = cache.shift_remove(&offset) { + // LRU + let res = BPlusTreeNode::::deserialize_from_block_slice(&data, file, false) .map_err(BPlusTreeError::from)?; - if let Some(removed) = cache.shift_remove(&offset) { - cache.insert(offset, removed); - } + cache.insert(offset, data); res } else { + // Disk Read match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { Ok((node, pointers)) => { + // Update Cache if cache.len() >= CACHE_CAPACITY { cache.shift_remove_index(0); } @@ -1820,14 +1892,10 @@ where let mut count = 0; let mut stack = vec![start_offset]; while let Some(offset) = stack.pop() { - let (node, pointers) = if let Some(cached_block) = cache.get_mut(&offset) { - let data = cached_block.clone(); - let temp_buffer = data; - let res = BPlusTreeNode::::deserialize_from_block_slice(&temp_buffer, file, false) + let (node, pointers) = if let Some(data) = cache.shift_remove(&offset) { + let res = BPlusTreeNode::::deserialize_from_block_slice(&data, file, false) .map_err(BPlusTreeError::from)?; - if let Some(removed) = cache.shift_remove(&offset) { - cache.insert(offset, removed); - } + cache.insert(offset, data); res } else { match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { @@ -1851,12 +1919,119 @@ where Ok(count) } +fn count_items_mmap( + mmap: &[u8], + start_offset: u64, +) -> Result +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut count = 0; + let mut stack = vec![start_offset]; + let mut cursor = io::Cursor::new(mmap); + while let Some(offset) = stack.pop() { + let (node, pointers) = BPlusTreeNode::::deserialize_from_mmap(mmap, &mut cursor, offset, false)?; + + if node.is_leaf { + count += node.keys.len(); + } else if let Some(ptrs) = pointers { + stack.extend(ptrs); + } + } + Ok(count) +} + +fn query_tree_mmap( + mmap: &[u8], + cursor: &mut io::Cursor<&[u8]>, + key: &K, + start_offset: u64, +) -> Result, BPlusTreeError> +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut offset = start_offset; + loop { + let (node, pointers) = BPlusTreeNode::::deserialize_from_mmap(mmap, cursor, offset, false)?; + + if node.is_leaf { + return match node.keys.binary_search(key) { + Ok(idx) => { + match node.value_info.get(idx) { + Some(info) => { + let value = BPlusTreeNode::::load_value_from_info(cursor, info)?; + Ok(Some(value)) + } + None => Ok(None), + } + } + Err(_) => Ok(None), + }; + } + + let child_idx = get_entry_index_upper_bound::(&node.keys, key); + if let Some(child_offsets) = pointers { + if let Some(child_offset) = child_offsets.get(child_idx) { + offset = *child_offset; + } else { + return Ok(None); + } + } else { + return Ok(None); + } + } +} + +fn query_tree_le_mmap( + mmap: &[u8], + cursor: &mut io::Cursor<&[u8]>, + key: &K, + start_offset: u64, +) -> Result, BPlusTreeError> +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut offset = start_offset; + loop { + let (node, pointers) = BPlusTreeNode::::deserialize_from_mmap(mmap, cursor, offset, false)?; + + if node.is_leaf { + let idx = get_entry_index_upper_bound::(&node.keys, key); + if idx == 0 { + return Ok(None); + } + return match node.value_info.get(idx - 1) { + Some(info) => { + let value = BPlusTreeNode::::load_value_from_info(cursor, info)?; + Ok(Some(value)) + } + None => Ok(None), + }; + } + + let child_idx = get_entry_index_upper_bound::(&node.keys, key); + if let Some(child_offsets) = pointers { + if let Some(child_offset) = child_offsets.get(child_idx) { + offset = *child_offset; + } else { + return Ok(None); + } + } else { + return Ok(None); + } + } +} + /// `BPlusTreeQuery` performs on-disk queries without loading the entire tree into memory. /// For frequent queries, consider using `BPlusTree::load()` instead, which loads the full tree into memory /// at the cost of higher memory usage. pub struct BPlusTreeQuery { - file: BufReader, + file: Option>, + mmap: Option, filepath: PathBuf, buffer: Vec, cache: IndexMap>, @@ -1870,12 +2045,28 @@ where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { - pub fn try_from_file(mut file: File) -> io::Result { + pub fn try_from_file(file: File) -> io::Result { + let metadata = file.metadata()?; + let file_len = metadata.len(); + + if file_len < HEADER_SIZE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "File too small")); + } + + // Try Mmap + let mmap = unsafe { Mmap::map(&file).ok() }; // Verify Header let mut header = [0u8; 16]; - file.seek(SeekFrom::Start(0))?; - file.read_exact(&mut header)?; + #[cfg(unix)] + file.read_exact_at(&mut header, 0)?; + #[cfg(not(unix))] + { + let mut f = &file; + f.seek(SeekFrom::Start(0))?; + f.read_exact(&mut header)?; + } + if &header[0..4] != MAGIC { return Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid magic number")); } @@ -1886,8 +2077,9 @@ where let root_offset = u64::from_le_bytes(header[8..16].try_into().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid root offset slice"))?); Ok(Self { - file: utils::file_reader(file), - filepath: PathBuf::new(), // Unknown when created from file directly + file: if mmap.is_some() { None } else { Some(utils::file_reader(file)) }, + mmap, + filepath: PathBuf::new(), buffer: vec![0u8; PAGE_SIZE_USIZE], cache: IndexMap::with_capacity(CACHE_CAPACITY), root_offset, @@ -1909,25 +2101,47 @@ where } pub fn query(&mut self, key: &K) -> Result, BPlusTreeError> { - query_tree(&mut self.file, &mut self.buffer, &mut self.cache, key, self.root_offset) - } - - pub fn is_empty(&mut self) -> Result { - let (node, _) = BPlusTreeNode::::deserialize_from_block( - &mut self.file, - &mut self.buffer, - self.root_offset, - false, - )?; - Ok(node.is_leaf && node.keys.is_empty()) - } - - pub fn len(&mut self) -> Result { - count_items::(&mut self.file, &mut self.buffer, &mut self.cache, self.root_offset) + if let Some(mmap) = &self.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + query_tree_mmap(mmap, &mut cursor, key, self.root_offset) + } else if let Some(file) = &mut self.file { + query_tree(file, &mut self.buffer, &mut self.cache, key, self.root_offset) + } else { + Err(BPlusTreeError::InvalidStructure("No data source available".into())) + } } pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { - query_tree_le(&mut self.file, &mut self.buffer, &mut self.cache, key, self.root_offset) + if let Some(mmap) = &self.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + query_tree_le_mmap(mmap, &mut cursor, key, self.root_offset) + } else if let Some(file) = &mut self.file { + query_tree_le(file, &mut self.buffer, &mut self.cache, key, self.root_offset) + } else { + Err(BPlusTreeError::InvalidStructure("No data source available".into())) + } + } + + pub fn len(&mut self) -> Result { + if let Some(mmap) = &self.mmap { + count_items_mmap::(mmap, self.root_offset) + } else if let Some(file) = &mut self.file { + count_items::(file, &mut self.buffer, &mut self.cache, self.root_offset) + } else { + Err(BPlusTreeError::InvalidStructure("No data source available".into())) + } + } + + pub fn is_empty(&mut self) -> Result { + let (node, _) = if let Some(mmap) = &self.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + BPlusTreeNode::::deserialize_from_mmap(mmap, &mut cursor, self.root_offset, false)? + } else if let Some(file) = &mut self.file { + BPlusTreeNode::::deserialize_from_block(file, &mut self.buffer, self.root_offset, false)? + } else { + return Err(BPlusTreeError::InvalidStructure("No data source available".into())); + }; + Ok(node.is_leaf && node.keys.is_empty()) } @@ -1935,9 +2149,9 @@ where pub fn iter(&mut self) -> BPlusTreeDiskIterator<'_, K, V> { BPlusTreeDiskIterator::new(self) } - + /// Owned iterator that traverses the tree in order defined by a secondary sorted index. - /// + /// /// The index path is automatically derived from the tree filepath by changing /// the extension to `.idx`. For example, if the tree is at `/data/items.bin`, /// the index is expected at `/data/items.idx`. @@ -1948,7 +2162,7 @@ where where SortKey: for<'de> Deserialize<'de>, { - super::sorted_index::BPlusTreeSortedIteratorOwned::new(self.filepath.clone(), self.file) + super::sorted_index::BPlusTreeSortedIteratorOwned::::new_hybrid(self.filepath.clone(), self.file, self.mmap) } /// Owned iterator with explicit index path. @@ -1956,7 +2170,7 @@ where where SortKey: for<'de> Deserialize<'de>, { - super::sorted_index::BPlusTreeSortedIteratorOwned::with_index_path(self.filepath.clone(), self.file, index_path) + super::sorted_index::BPlusTreeSortedIteratorOwned::::with_index_path_hybrid(self.filepath.clone(), self.file, self.mmap, index_path) } /// Traverses the tree and calls the provided closure for each leaf's keys and values. @@ -1978,43 +2192,44 @@ where pub fn collect_with_locations(&mut self) -> io::Result> { let mut result = Vec::new(); let mut stack = vec![self.root_offset]; - + while let Some(offset) = stack.pop() { - let (node, pointers) = BPlusTreeNode::::deserialize_from_block( - &mut self.file, - &mut self.buffer, - offset, - false, - )?; - + let (node, pointers) = if let Some(mmap) = &self.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + BPlusTreeNode::::deserialize_from_mmap(mmap, &mut cursor, offset, false)? + } else if let Some(file) = &mut self.file { + BPlusTreeNode::::deserialize_from_block(file, &mut self.buffer, offset, false)? + } else { + return Err(io::Error::other("No data source available")); + }; + if node.is_leaf { - for (key, info) in node.keys.into_iter().zip(node.value_info.iter()) { - let value = BPlusTreeNode::::load_value_from_info(&mut self.file, info)?; - let location = match info.mode { - ValueStorageMode::Single(offset) => { - super::sorted_index::ValueLocation::Single { - offset, - length: info.length, - } - } - ValueStorageMode::Packed(block_offset, index) => { - super::sorted_index::ValueLocation::Packed { - block_offset, - index, - length: info.length, - } - } - }; - result.push((key, value, location)); + if let Some(mmap) = &self.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + for (key, info) in node.keys.into_iter().zip(node.value_info.iter()) { + let value = BPlusTreeNode::::load_value_from_info(&mut cursor, info)?; + let location = match info.mode { + ValueStorageMode::Single(offset) => super::sorted_index::ValueLocation::Single { offset, length: info.length }, + ValueStorageMode::Packed(block_offset, index) => super::sorted_index::ValueLocation::Packed { block_offset, index, length: info.length }, + }; + result.push((key, value, location)); + } + } else if let Some(file) = &mut self.file { + for (key, info) in node.keys.into_iter().zip(node.value_info.iter()) { + let value = BPlusTreeNode::::load_value_from_info(file, info)?; + let location = match info.mode { + ValueStorageMode::Single(offset) => super::sorted_index::ValueLocation::Single { offset, length: info.length }, + ValueStorageMode::Packed(block_offset, index) => super::sorted_index::ValueLocation::Packed { block_offset, index, length: info.length }, + }; + result.push((key, value, location)); + } } } else if let Some(ptrs) = pointers { - // Process children in reverse order so we pop them in correct order for ptr in ptrs.into_iter().rev() { stack.push(ptr); } } } - Ok(result) } @@ -2022,7 +2237,6 @@ where pub fn disk_iter(self) -> BPlusTreeDiskIteratorOwned { BPlusTreeDiskIteratorOwned::new(self) } - } @@ -2058,18 +2272,28 @@ where loop { let Some((offset, child_idx)) = self.stack.pop() else { return Ok(None) }; - let (node, pointers) = BPlusTreeNode::::deserialize_from_block( - &mut self.query.file, - &mut self.query.buffer, - offset, - false, - )?; + let (node, pointers) = if let Some(mmap) = &self.query.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + BPlusTreeNode::::deserialize_from_mmap(mmap, &mut cursor, offset, false)? + } else if let Some(file) = &mut self.query.file { + BPlusTreeNode::::deserialize_from_block(file, &mut self.query.buffer, offset, false)? + } else { + return Err(io::Error::other("No data source available")); + }; if node.is_leaf { let mut vals = Vec::with_capacity(node.value_info.len()); - for value_info in &node.value_info { - let v = BPlusTreeNode::::load_value_from_info(&mut self.query.file, value_info)?; - vals.push(v); + if let Some(mmap) = &self.query.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + for value_info in &node.value_info { + let v = BPlusTreeNode::::load_value_from_info(&mut cursor, value_info)?; + vals.push(v); + } + } else if let Some(file) = &mut self.query.file { + for value_info in &node.value_info { + let v = BPlusTreeNode::::load_value_from_info(file, value_info)?; + vals.push(v); + } } return Ok(Some((node.keys, vals))); } else if let Some(pters) = pointers { @@ -2165,18 +2389,28 @@ where loop { let Some((offset, child_idx)) = self.stack.pop() else { return Ok(None) }; - let (node, pointers) = BPlusTreeNode::::deserialize_from_block( - &mut self.query.file, - &mut self.query.buffer, - offset, - false, - )?; + let (node, pointers) = if let Some(mmap) = &self.query.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + BPlusTreeNode::::deserialize_from_mmap(mmap, &mut cursor, offset, false)? + } else if let Some(file) = &mut self.query.file { + BPlusTreeNode::::deserialize_from_block(file, &mut self.query.buffer, offset, false)? + } else { + return Err(io::Error::other("No data source available")); + }; if node.is_leaf { let mut vals = Vec::with_capacity(node.value_info.len()); - for value_info in &node.value_info { - let v = BPlusTreeNode::::load_value_from_info(&mut self.query.file, value_info)?; - vals.push(v); + if let Some(mmap) = &self.query.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + for value_info in &node.value_info { + let v = BPlusTreeNode::::load_value_from_info(&mut cursor, value_info)?; + vals.push(v); + } + } else if let Some(file) = &mut self.query.file { + for value_info in &node.value_info { + let v = BPlusTreeNode::::load_value_from_info(file, value_info)?; + vals.push(v); + } } return Ok(Some((node.keys, vals))); } else if let Some(pters) = pointers { @@ -2223,7 +2457,7 @@ where } pub struct BPlusTreeUpdate { - file: File, + file: BufReader, read_buffer: Vec, write_buffer: Vec, cache: IndexMap>, @@ -2290,7 +2524,8 @@ where // Acquire lock first let lock = FileLock::try_lock(filepath)?; - let mut file = utils::open_read_write_file(filepath)?; + let file = utils::open_read_write_file(filepath)?; + let mut file = utils::file_reader(file); // Verify Header let mut header = [0u8; 16]; @@ -2311,14 +2546,51 @@ where write_buffer: vec![0u8; PAGE_SIZE_USIZE], cache: IndexMap::with_capacity(CACHE_CAPACITY), root_offset, - lock, inner_order, leaf_order, + lock, _marker_k: PhantomData, _marker_v: PhantomData, }) } + /// Helper to access metadata + pub fn get_metadata(&mut self) -> io::Result { + self.file.seek(SeekFrom::Start(METADATA_OFFSET_POS))?; + let mut len_buf = [0u8; 4]; + if self.file.read_exact(&mut len_buf).is_err() { + return Ok(BPlusTreeMetadata::Empty); + } + let len = u32::from_le_bytes(len_buf); + if len == 0 || len as usize > METADATA_MAX_SIZE { + return Ok(BPlusTreeMetadata::Empty); + } + let mut buf = vec![0u8; len as usize]; + self.file.read_exact(&mut buf)?; + + Ok(BPlusTreeMetadata::from_bytes(&buf)) + } + + /// Helper to set metadata + pub fn set_metadata(&mut self, data: &BPlusTreeMetadata) -> io::Result<()> { + let bytes = data.to_bytes(); + if bytes.len() > METADATA_MAX_SIZE { + return Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Metadata too large: {} > {}", bytes.len(), METADATA_MAX_SIZE))); + } + + self.file.seek(SeekFrom::Start(METADATA_OFFSET_POS))?; + { + let file = self.file.get_mut(); + file.write_all(&u32::try_from(bytes.len()).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?.to_le_bytes())?; + file.write_all(&bytes)?; + file.flush()?; // Ensure it hits disk + } + // Resync/clear reader buffer after direct writes. + self.file.seek_relative(0)?; // same as self.file.seek(SeekFrom::Current(0))?; + + Ok(()) + } + pub fn query(&mut self, key: &K) -> Result, BPlusTreeError> { let mut reader = utils::file_reader(&mut self.file); query_tree(&mut reader, &mut self.read_buffer, &mut self.cache, key, self.root_offset) @@ -2336,13 +2608,11 @@ where } pub fn len(&mut self) -> Result { - let mut reader = utils::file_reader(&mut self.file); - count_items::(&mut reader, &mut self.read_buffer, &mut self.cache, self.root_offset) + count_items::(&mut self.file, &mut self.read_buffer, &mut self.cache, self.root_offset) } pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { - let mut reader = utils::file_reader(&mut self.file); - query_tree_le(&mut reader, &mut self.read_buffer, &mut self.cache, key, self.root_offset) + query_tree_le(&mut self.file, &mut self.read_buffer, &mut self.cache, key, self.root_offset) } @@ -2351,10 +2621,10 @@ where let new_root_offset = self.update_batch_recursive(self.root_offset, &refs)?; // Atomic Header Swap - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; - self.file.write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; - self.file.flush().map_err(BPlusTreeError::Io)?; - self.file.sync_all().map_err(BPlusTreeError::Io)?; + self.file.get_mut().seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; + self.file.get_mut().write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; + self.file.get_mut().flush().map_err(BPlusTreeError::Io)?; + self.file.get_mut().sync_all().map_err(BPlusTreeError::Io)?; self.root_offset = new_root_offset; Ok(new_root_offset) @@ -2368,8 +2638,7 @@ where offset: u64, items: &[(&K, &V)], ) -> Result { - let mut reader = utils::file_reader(&mut self.file); - let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false)?; + let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block(&mut self.file, &mut self.read_buffer, offset, false)?; if node.is_leaf { for (key, value) in items { @@ -2426,10 +2695,10 @@ where let new_root_offset = self.update_batch_recursive(self.root_offset, &sorted_items)?; // Atomic Header Swap - only once at the end - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; - self.file.write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; - self.file.flush().map_err(BPlusTreeError::Io)?; - self.file.sync_all().map_err(BPlusTreeError::Io)?; + self.file.get_mut().seek(SeekFrom::Start(ROOT_OFFSET_POS)).map_err(BPlusTreeError::Io)?; + self.file.get_mut().write_all(&new_root_offset.to_le_bytes()).map_err(BPlusTreeError::Io)?; + self.file.get_mut().flush().map_err(BPlusTreeError::Io)?; + self.file.get_mut().sync_all().map_err(BPlusTreeError::Io)?; self.root_offset = new_root_offset; Ok(new_root_offset) @@ -2446,13 +2715,13 @@ where // Decide whether to compress based on size and effectiveness let (flag, payload) = compress_if_beneficial(&raw_bytes); - self.file.seek(SeekFrom::End(0))?; - let offset = self.file.stream_position()?; + self.file.get_mut().seek(SeekFrom::End(0))?; + let offset = self.file.get_mut().stream_position()?; // Write: [flag:1][payload] - self.file.write_all(&[flag])?; + self.file.get_mut().write_all(&[flag])?; // NOTE: For LZ4, payload already includes original length (prepended) - self.file.write_all(&payload)?; + self.file.get_mut().write_all(&payload)?; // stored_len includes flag + payload let stored_len = 1 + payload.len(); @@ -2460,16 +2729,16 @@ where } fn write_node(&mut self, node: &BPlusTreeNode) -> io::Result { - self.file.seek(SeekFrom::End(0))?; - let offset = self.file.stream_position()?; - node.serialize_to_block(&mut self.file, &mut self.write_buffer, offset)?; + self.file.get_mut().seek(SeekFrom::End(0))?; + let offset = self.file.get_mut().stream_position()?; + node.serialize_to_block(self.file.get_mut(), &mut self.write_buffer, offset)?; Ok(offset) } fn write_internal_node(&mut self, node: &BPlusTreeNode, pointers: &[u64]) -> io::Result { - self.file.seek(SeekFrom::End(0))?; - let offset = self.file.stream_position()?; - node.serialize_internal_with_offsets(&mut self.file, &mut self.write_buffer, offset, pointers)?; + self.file.get_mut().seek(SeekFrom::End(0))?; + let offset = self.file.get_mut().stream_position()?; + node.serialize_internal_with_offsets(self.file.get_mut(), &mut self.write_buffer, offset, pointers)?; Ok(offset) } @@ -2479,9 +2748,8 @@ where offset: u64, items: &[(&K, &V)], ) -> io::Result<(u64, Vec<(K, u64)>)> { - let mut reader = utils::file_reader(&mut self.file); let (mut node, pointers_opt) = BPlusTreeNode::::deserialize_from_block( - &mut reader, + &mut self.file, &mut self.read_buffer, offset, false, // shallow @@ -2607,10 +2875,10 @@ where // Handle promotions (splits) using balanced approach current_root = self.build_higher_levels(current_root, promotions)?; - self.file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; - self.file.write_all(¤t_root.to_le_bytes())?; - self.file.flush()?; - self.file.sync_all()?; + self.file.get_mut().seek(SeekFrom::Start(ROOT_OFFSET_POS))?; + self.file.get_mut().write_all(¤t_root.to_le_bytes())?; + self.file.get_mut().flush()?; + self.file.get_mut().sync_all()?; self.root_offset = current_root; Ok(current_root) @@ -2618,25 +2886,62 @@ where /// Garbage Collection: Compacts the file by rewriting only live blocks sequentially. pub fn compact(&mut self, filepath: &Path) -> io::Result<()> { - let mut value_infos: Vec<(K, ValueInfo)> = Vec::new(); let mut temp_file = NamedTempFile::new_in(filepath.parent().unwrap_or(Path::new(".")))?; - // 1. Write Header placeholder + // 1. Read existing metadata from source (manually to avoid full load) + let mut metadata = Vec::new(); + { + if let Ok(mut src) = File::open(filepath) { + if src.seek(SeekFrom::Start(METADATA_OFFSET_POS)).is_ok() { + let mut lbuf = [0u8; 4]; + if src.read_exact(&mut lbuf).is_ok() { + let l = u32::from_le_bytes(lbuf); + if l > 0 && l as usize <= METADATA_MAX_SIZE { + let mut b = vec![0u8; l as usize]; + if src.read_exact(&mut b).is_ok() { + metadata = b; + } else { + error!("Failed to read metadata bytes during compaction"); + } + } else if l > u32::try_from(METADATA_MAX_SIZE).unwrap_or(4000) { + error!("Metadata too large during compaction: {l}"); + } + } + } + } + } + + // 2. Write Header placeholder temp_file.seek(SeekFrom::Start(0))?; - temp_file.write_all(MAGIC)?; - temp_file.write_all(&STORAGE_VERSION.to_le_bytes())?; - temp_file.seek(SeekFrom::Start(HEADER_SIZE))?; // Skip to start of content + // Construct full header block + let mut header = [0u8; PAGE_SIZE_USIZE]; + header[0..4].copy_from_slice(MAGIC); + header[4..8].copy_from_slice(&STORAGE_VERSION.to_le_bytes()); + // Root offset placeholder (will be filled later) + // Root offset placeholder (will be filled later) + if !metadata.is_empty() { + let len = u32::try_from(metadata.len()).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + // Ensure we check max size? + header[16..20].copy_from_slice(&len.to_le_bytes()); + header[20..20 + metadata.len()].copy_from_slice(&metadata); + } + temp_file.write_all(&header)?; + + // temp_file.seek(SeekFrom::Start(HEADER_SIZE))?; // Skip to start of content (implied by write_all(4096)) let mut current_offset = HEADER_SIZE; + let mut leaf_pointers: Vec<(K, u64)> = Vec::new(); + let mut current_leaf = BPlusTreeNode::::new(true); - // 2. Iterate source and write values immediately (Streaming) + // 2. Iterate source and write values + leaf nodes immediately (Streaming) { let mut query = BPlusTreeQuery::::try_new(filepath).map_err(to_io_error)?; let mut write_buffer = std::io::BufWriter::new(&mut temp_file); + let mut node_buffer = vec![0u8; PAGE_SIZE_USIZE]; for (k, v) in query.iter() { let value_bytes = binary_serialize(&v)?; - let offset = current_offset; + let val_offset = current_offset; // Write value header (flag + payload) let (flag, payload) = compress_if_beneficial(&value_bytes); @@ -2646,20 +2951,36 @@ where let stored_len = u32::try_from(1 + payload.len()).map_err(to_io_error)?; current_offset += u64::from(stored_len); - value_infos.push((k, ValueInfo { - mode: ValueStorageMode::Single(offset), + current_leaf.keys.push(k); + current_leaf.value_info.push(ValueInfo { + mode: ValueStorageMode::Single(val_offset), length: stored_len, compressed_cache: None, - })); + }); + + if current_leaf.keys.len() >= self.leaf_order { + let first_key = current_leaf.keys[0].clone(); + let node_offset = current_offset; + current_offset = current_leaf.serialize_to_block(&mut write_buffer, &mut node_buffer, node_offset)?; + leaf_pointers.push((first_key, node_offset)); + current_leaf = BPlusTreeNode::new(true); + } + } + + // Handle trailing leaf + if !current_leaf.keys.is_empty() { + let first_key = current_leaf.keys[0].clone(); + let node_offset = current_offset; + current_offset = current_leaf.serialize_to_block(&mut write_buffer, &mut node_buffer, node_offset)?; + leaf_pointers.push((first_key, node_offset)); } write_buffer.flush()?; } - // 3. Build Tree Structure - current_offset = temp_file.seek(SeekFrom::End(0))?; - - let mut tree = BPlusTree::::new(); - let root_offset = tree.bulk_build_from_infos(&mut temp_file, &value_infos, current_offset)?; + // 3. Build Internal Levels + let tree = BPlusTree::::new(); + let mut node_buffer = vec![0u8; PAGE_SIZE_USIZE]; + let root_offset = tree.build_levels_from_pointers(&mut temp_file, leaf_pointers, current_offset, &mut node_buffer)?; // 4. Update Header temp_file.seek(SeekFrom::Start(ROOT_OFFSET_POS))?; @@ -2673,7 +2994,8 @@ where // 6. Refresh state self.root_offset = root_offset; - self.file = utils::open_read_write_file(filepath)?; + let file = utils::open_read_write_file(filepath)?; + self.file = utils::file_reader(file); self.cache.clear(); Ok(()) diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index 4a7bc91e1..50972cc3a 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -44,7 +44,7 @@ pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path, playl for channel in &group.channels { if let Some(epg_id) = &channel.header.epg_channel_id { if !epg_id.is_empty() { - rename_map.insert(epg_id.as_str(), channel.header.name.as_str()); + rename_map.insert(epg_id, &channel.header.name); } } } diff --git a/backend/src/repository/library_repository.rs b/backend/src/repository/library_repository.rs index 47b95453d..1e1ba1161 100644 --- a/backend/src/repository/library_repository.rs +++ b/backend/src/repository/library_repository.rs @@ -1,10 +1,11 @@ use crate::model::AppConfig; use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; -use shared::error::{TuliproxError, notify_err_res}; -use shared::model::{PlaylistGroup, PlaylistItem, UUIDType, XtreamCluster, XtreamPlaylistItem}; +use shared::error::{notify_err_res, TuliproxError}; +use shared::model::{PlaylistGroup, PlaylistItem, XtreamCluster, XtreamPlaylistItem}; use std::path::Path; use std::sync::Arc; use indexmap::IndexMap; +use shared::model::UUIDType; use crate::repository::xtream_repository::CategoryKey; pub async fn persist_input_library_playlist(app_config: &Arc, library_path: &Path, playlist: &[PlaylistGroup]) -> Result<(), TuliproxError> { diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index 56aa2ce5b..c3c78f841 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -12,7 +12,7 @@ use crate::utils::FileReadGuard; use std::collections::HashSet; use std::iter::Peekable; use log::error; -// concat_string! macro from shared utils is used for efficient String building +use shared::utils::Internable; #[allow(clippy::struct_excessive_bools)] pub struct M3uPlaylistIterator { @@ -195,7 +195,7 @@ impl M3uPlaylistIterator { } else { None }; - m3u_pli.t_stream_url = stream_url; + m3u_pli.t_stream_url = stream_url.intern(); m3u_pli.t_resource_url = resource_url; } else { // Keep original URL (clone required because target field is distinct) diff --git a/backend/src/repository/m3u_repository.rs b/backend/src/repository/m3u_repository.rs index 5e7df23b3..b868dce50 100644 --- a/backend/src/repository/m3u_repository.rs +++ b/backend/src/repository/m3u_repository.rs @@ -1,22 +1,24 @@ use crate::api::model::AppState; -use crate::model::{AppConfig, ProxyUserCredentials}; +use crate::model::{AppConfig, ConfigInput, ProxyUserCredentials}; use crate::model::{Config, ConfigTarget, M3uTargetOutput}; use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; use crate::repository::m3u_playlist_iterator::M3uPlaylistM3uTextIterator; -use crate::repository::storage::{get_file_path_for_db_index, get_target_storage_path}; +use crate::repository::playlist_repository::get_input_m3u_playlist_file_path; +use crate::repository::storage::{get_input_storage_path, get_target_storage_path}; use crate::repository::storage_const; use crate::repository::xtream_repository::CategoryKey; use crate::utils; -use crate::utils::{async_file_writer, IO_BUFFER_SIZE}; +use crate::utils::{async_file_writer, FileReadGuard, IO_BUFFER_SIZE}; use indexmap::IndexMap; use log::error; use shared::concat_string; -use shared::error::{info_err, notify_err, str_to_io_error, string_to_io_error, TuliproxError}; +use shared::error::{notify_err, str_to_io_error, string_to_io_error, TuliproxError}; use shared::model::{M3uPlaylistItem, PlaylistGroup}; use shared::model::{PlaylistItem, PlaylistItemType, XtreamCluster}; use std::io::Error; use std::path::{Path, PathBuf}; use std::sync::Arc; +use serde::{Deserialize, Serialize}; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio::task; @@ -158,41 +160,49 @@ pub async fn m3u_get_item_for_stream_id(stream_id: u32, app_state: &AppState, ta } } -pub async fn iter_raw_m3u_playlist(config: &AppConfig, target: &ConfigTarget) -> Option<(utils::FileReadGuard, impl Iterator)> { +pub async fn iter_raw_m3u_target_playlist(config: &AppConfig, target: &ConfigTarget, cluster: Option) -> Option<(FileReadGuard, Box + Send>)> { let target_path = get_target_storage_path(&config.config.load(), target.name.as_str())?; let m3u_path = m3u_get_file_path_for_db(&target_path); - if let Ok(false) = tokio::fs::try_exists(&m3u_path).await { + + iter_raw_m3u_playlist::(config, &m3u_path, cluster).await +} + +pub async fn iter_raw_m3u_input_playlist(app_config: &AppConfig, input: &ConfigInput, cluster: Option) -> Option<(FileReadGuard, Box + Send>)> { + let working_dir = &app_config.config.load().working_dir; + let storage_path = get_input_storage_path(&input.name, working_dir).ok()?; + let m3u_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); + + iter_raw_m3u_playlist::>(app_config, &m3u_path, cluster).await +} + +async fn iter_raw_m3u_playlist(app_config: &AppConfig, m3u_path: &Path, cluster: Option) -> Option<(FileReadGuard, Box + Send>)> +where + ItemKey: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Send + 'static, + SortKey: for<'de> Deserialize<'de> + Send + 'static, +{ + let file_lock = app_config.file_locks.read_lock(m3u_path).await; + if !tokio::fs::try_exists(m3u_path).await.unwrap_or(false) { return None; } - let file_lock = config.file_locks.read_lock(&m3u_path).await; - match BPlusTreeQuery::::try_new(&m3u_path) - .map_err(|err| info_err!("Could not open BPlusTreeQuery {m3u_path:?} - {err}")) { - Ok(mut query) => { - let index_path = get_file_path_for_db_index(&m3u_path); - let items: Vec = if index_path.exists() { - match query.disk_iter_sorted::() { - Ok(iter) => iter.filter_map(Result::ok).map(|(_, v)| v).collect(), - Err(err) => { - error!("Sorted index error {}: {err}", m3u_path.display()); - // Re-open query for fallback - match BPlusTreeQuery::::try_new(&m3u_path) { - Ok(mut query) => query.iter().map(|(_, v)| v).collect(), - Err(fallback_err) => { - error!("Fallback query also failed {}: {fallback_err}", m3u_path.display()); - Vec::new() - } - } + + let iter: Box + Send> = { + match BPlusTreeQuery::::try_new(m3u_path) { + Ok(tree) => match tree.disk_iter_sorted::() { + Ok(sorted_iter) => Box::new(sorted_iter.filter_map(Result::ok).map(|(_, v)| v).filter(move |v| cluster.is_none_or(|c| v.item_type.is_cluster(c)))), + Err(_) => { + match BPlusTreeQuery::::try_new(m3u_path) { + Ok(tree) => Box::new(tree.disk_iter().map(|(_, v)| v).filter(move |v| cluster.is_none_or(|c| v.item_type.is_cluster(c)))), + Err(_) => return None, } } - } else { - query.iter().map(|(_, v)| v).collect() - }; - - let len = items.len(); - Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i + 1 < len)))) + } + Err(_) => { + return None + } } - Err(_) => None - } + }; + + Some((file_lock, iter)) } pub async fn persist_input_m3u_playlist(app_config: &Arc, m3u_path: &Path, playlist: &[PlaylistGroup]) -> Result<(), TuliproxError> { @@ -226,7 +236,7 @@ pub async fn load_input_m3u_playlist(app_config: &Arc, m3u_path: &Pat if tokio::fs::try_exists(m3u_path).await.unwrap_or(false) { // Load Items let _file_lock = app_config.file_locks.read_lock(m3u_path).await; - if let Ok(mut query) = BPlusTreeQuery::::try_new(m3u_path) { + if let Ok(mut query) = BPlusTreeQuery::, M3uPlaylistItem>::try_new(m3u_path) { let mut group_cnt = 0; for (_, ref item) in query.iter() { let cluster = XtreamCluster::try_from(item.item_type).unwrap_or(XtreamCluster::Live); diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 43ca14308..4056d8bec 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -16,14 +16,14 @@ use crate::repository::{LocalLibraryDiskPlaylistSource, M3uDiskPlaylistSource, M use shared::error::{info_err, TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{InputType, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{is_dash_url, is_hls_url, StringInterner}; +use shared::utils::{is_dash_url, is_hls_url, Internable}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::repository::library_repository::{load_input_local_library_playlist, persist_input_library_playlist}; struct LocalEpisodeKey { - path: String, + path: Arc, virtual_id: u32, } @@ -33,8 +33,7 @@ pub struct ProviderEpisodeKey { } pub async fn persist_playlist(app_config: &Arc, playlist: &mut [PlaylistGroup], epg: Option<&Epg>, - target: &ConfigTarget, playlist_state: Option<&Arc>, - interner: &mut StringInterner) -> Result<(), Vec> { + target: &ConfigTarget, playlist_state: Option<&Arc>) -> Result<(), Vec> { let mut errors = vec![]; let config = &app_config.config.load(); let target_path = match ensure_target_storage_path(config, &target.name) { @@ -42,10 +41,13 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl Err(err) => return Err(vec![err]), }; - let (mut target_id_mapping, file_lock) = get_target_id_mapping(app_config, &target_path).await; + let (mut target_id_mapping, file_lock) = match get_target_id_mapping(app_config, &target_path, target.use_memory_cache).await { + Ok(result) => result, + Err(err) => return Err(vec![err]), + }; - let mut local_library_series = HashMap::>::new(); - let mut provider_series = HashMap::>::new(); + let mut local_library_series = HashMap::, Vec>::new(); + let mut provider_series = HashMap::, Vec>::new(); let mut source_ordinal: u32 = 0; // Virtual IDs assignment @@ -98,7 +100,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl }; let result = match output { - TargetOutput::Xtream(_xtream_output) => xtream_write_playlist(app_config, target, pl, interner).await, + TargetOutput::Xtream(_xtream_output) => xtream_write_playlist(app_config, target, pl).await, TargetOutput::M3u(m3u_output) => m3u_write_playlist(app_config, target, m3u_output, &target_path, pl).await, TargetOutput::Strm(strm_output) => write_strm_playlist(app_config, target, strm_output, pl).await, TargetOutput::HdHomeRun(_hdhomerun_output) => Ok(()), @@ -120,6 +122,9 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl if let Err(err) = target_id_mapping.persist() { errors.push(info_err!("{err}")); } + // We must drop target_id_mapping here to release the exclusive B+Tree lock + // otherwise the subsequent load_xtream_target_storage will deadlock waiting for a shared lock. + drop(target_id_mapping); drop(file_lock); if target.use_memory_cache { @@ -145,7 +150,7 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl if errors.is_empty() { Ok(()) } else { Err(errors) } } -fn assign_local_series_info_episode_key(local_library_series: &mut HashMap>, header: &mut PlaylistItemHeader, item_type: PlaylistItemType) { +fn assign_local_series_info_episode_key(local_library_series: &mut HashMap, Vec>, header: &mut PlaylistItemHeader, item_type: PlaylistItemType) { // we need to rewrite local series info with the new virtual ids if item_type == PlaylistItemType::LocalSeries { local_library_series @@ -158,7 +163,7 @@ fn assign_local_series_info_episode_key(local_library_series: &mut HashMap>, header: &mut PlaylistItemHeader, item_type: PlaylistItemType) { +fn assign_provider_series_info_episode_key(provider_series: &mut HashMap, Vec>, header: &mut PlaylistItemHeader, item_type: PlaylistItemType) { // we need to rewrite local series info with the new virtual ids if item_type == PlaylistItemType::Series { provider_series @@ -172,11 +177,11 @@ fn assign_provider_series_info_episode_key(provider_series: &mut HashMap>) { +fn rewrite_local_series_info_episode_virtual_id(pli: &mut PlaylistItem, local_library_series: &HashMap, Vec>) { let header = &mut pli.header; // the local_library_series key is the id of the SeriesInfo. The episodes have their parent SeriesInfo id as parent_code. // When we populate local_library_series, we use the episodes.parent_code. Here we need to use the SeriesInfo.id to get the assigned episodes. - if let Some(episode_keys) = local_library_series.get(&header.id) { + if let Some(episode_keys) = local_library_series.get(&*header.id) { if let Some(StreamProperties::Series(series)) = header.additional_properties.as_mut() { if let Some(episodes) = series.details.as_mut().and_then(|d| d.episodes.as_mut()) @@ -195,11 +200,11 @@ fn rewrite_local_series_info_episode_virtual_id(pli: &mut PlaylistItem, local_li } #[allow(clippy::implicit_hasher)] -pub fn rewrite_provider_series_info_episode_virtual_id

(pli: &mut P, provider_series: &HashMap>) +pub fn rewrite_provider_series_info_episode_virtual_id

(pli: &mut P, provider_series: &HashMap, Vec>) where P: PlaylistEntry, { - if let Some(episode_keys) = provider_series.get(&pli.get_uuid().to_string()) { + if let Some(episode_keys) = provider_series.get(&pli.get_uuid().intern()) { if let Some(StreamProperties::Series(series)) = pli.get_additional_properties_mut() { if let Some(episodes) = series.details.as_mut().and_then(|d| d.episodes.as_mut()) @@ -218,8 +223,8 @@ where } fn rewrite_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup], - local_library_series: &HashMap>, - provider_series: &HashMap>) { + local_library_series: &HashMap, Vec>, + provider_series: &HashMap, Vec>) { if local_library_series.is_empty() && provider_series.is_empty() { return; } @@ -231,16 +236,17 @@ fn rewrite_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup], } else if item_type == PlaylistItemType::LocalSeriesInfo { rewrite_local_series_info_episode_virtual_id(channel, local_library_series); } else if item_type == PlaylistItemType::LocalSeries { - channel.header.parent_code = String::new(); + channel.header.parent_code = "".intern(); } } } } -pub async fn get_target_id_mapping(cfg: &AppConfig, target_path: &Path) -> (TargetIdMapping, utils::FileWriteGuard) { +pub async fn get_target_id_mapping(cfg: &AppConfig, target_path: &Path, use_memory_cache: bool) -> Result<(TargetIdMapping, utils::FileWriteGuard), TuliproxError> { let target_id_mapping_file = get_target_id_mapping_file(target_path); let file_lock = cfg.file_locks.write_lock(&target_id_mapping_file).await; - (TargetIdMapping::new(&target_id_mapping_file), file_lock) + let mapping = TargetIdMapping::new(&target_id_mapping_file, use_memory_cache)?; + Ok((mapping, file_lock)) } @@ -355,7 +361,7 @@ pub async fn persist_input_playlist(app_config: &Arc, input: &ConfigI return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); } }; - let file_path = get_input_m3u_playlist_file_path(&storage_path, input.name.as_str()); + let file_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); if let Err(err) = persist_input_m3u_playlist(app_config, &file_path, &playlist).await { return (playlist, Some(err)); } @@ -370,7 +376,7 @@ pub async fn persist_input_playlist(app_config: &Arc, input: &ConfigI return (playlist, Some(info_err!("Error creating input storage directory for input '{}' failed: {err}", input.name))); } }; - let file_path = get_input_local_library_playlist_file_path(&storage_path, input.name.as_str()); + let file_path = get_input_local_library_playlist_file_path(&storage_path, &input.name); if let Err(err) = persist_input_library_playlist(app_config, &file_path, &playlist).await { return (playlist, Some(err)); } @@ -403,7 +409,7 @@ pub async fn load_input_playlist(ctx: &PlaylistProcessingContext, input: &Config } InputType::M3u | InputType::M3uBatch => { // Load M3U - let file_path = get_input_m3u_playlist_file_path(&storage_path, input.name.as_str()); + let file_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); if disk_based_processing && file_path.exists() { Ok(Box::new(M3uDiskPlaylistSource::new(app_config, &file_path).await)) } else { @@ -412,7 +418,7 @@ pub async fn load_input_playlist(ctx: &PlaylistProcessingContext, input: &Config } } InputType::Library => { - let file_path = get_input_local_library_playlist_file_path(&storage_path, input.name.as_str()); + let file_path = get_input_local_library_playlist_file_path(&storage_path, &input.name); if disk_based_processing && file_path.exists() { Ok(Box::new(LocalLibraryDiskPlaylistSource::new(app_config, &file_path).await)) } else { @@ -423,14 +429,14 @@ pub async fn load_input_playlist(ctx: &PlaylistProcessingContext, input: &Config } } -fn get_input_m3u_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { +pub fn get_input_m3u_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { let sanitized_input_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); storage_path.join(format!("m3u_{sanitized_input_name}.{FILE_SUFFIX_DB}")) } -fn get_input_local_library_playlist_file_path(storage_path: &Path, input_name: &str) -> PathBuf { +fn get_input_local_library_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { let sanitized_input_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); diff --git a/backend/src/repository/playlist_source.rs b/backend/src/repository/playlist_source.rs index 37135a057..0ab8af01f 100644 --- a/backend/src/repository/playlist_source.rs +++ b/backend/src/repository/playlist_source.rs @@ -6,11 +6,12 @@ use futures::future::BoxFuture; use indexmap::IndexMap; use log::{error, warn}; use serde::{Deserialize, Serialize}; -use shared::model::{M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, UUIDType, XtreamCluster, XtreamPlaylistItem}; +use shared::model::{M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use std::borrow::Cow; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; +use shared::model::UUIDType; pub trait PlaylistSource: Send + Sync { fn is_memory(&self) -> bool; diff --git a/backend/src/repository/sorted_index.rs b/backend/src/repository/sorted_index.rs index 3e0f59285..fabbeb3bb 100644 --- a/backend/src/repository/sorted_index.rs +++ b/backend/src/repository/sorted_index.rs @@ -26,15 +26,15 @@ //! > **Important**: The index is tightly coupled to the B+Tree file structure. //! > It must be rebuilt after any operation that changes value offsets (e.g., `compact()`). +use crate::repository::bplustree::{COMPRESSION_FLAG_LZ4, PAGE_SIZE_USIZE}; +use crate::repository::storage::get_file_path_for_db_index; use crate::utils::{binary_deserialize, binary_serialize}; +use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use std::fs::{File, OpenOptions}; use std::io::{self, BufReader, BufWriter, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; -use crate::repository::bplustree::{COMPRESSION_FLAG_LZ4, PAGE_SIZE_USIZE}; -use indexmap::IndexMap; -use crate::repository::storage::get_file_path_for_db_index; const MAGIC: &[u8; 4] = b"SIDX"; const VERSION: u32 = 3; // Bumped for new format with flexible value location @@ -44,7 +44,6 @@ const MODE_SINGLE: u8 = 0; const MODE_PACKED: u8 = 1; - /// Represents how a value can be located and read from the tree file. #[derive(Debug, Clone, Copy)] pub enum ValueLocation { @@ -321,7 +320,8 @@ where /// tree file using stored offsets - no tree traversal needed (O(1) per item). pub struct BPlusTreeSortedIteratorOwned { index_reader: SortedIndexReader, - tree_file: BufReader, + tree_file: Option>, + mmap: Option, filepath: PathBuf, block_cache: IndexMap>, _marker: PhantomData, @@ -339,12 +339,30 @@ where /// /// The index path is automatically derived from the tree filepath /// by changing the extension to `.idx`. - pub fn new(filepath: PathBuf, tree_file: BufReader) -> io::Result { + pub fn new(filepath: PathBuf, tree_file: Option>) -> io::Result { let index_path = get_file_path_for_db_index(&filepath); let index_reader = SortedIndexReader::open(&index_path)?; Ok(Self { index_reader, tree_file, + mmap: None, + filepath, + block_cache: IndexMap::with_capacity(CACHE_CAPACITY), + _marker: PhantomData, + }) + } + + pub fn new_hybrid( + filepath: PathBuf, + tree_file: Option>, + mmap: Option, + ) -> io::Result { + let index_path = get_file_path_for_db_index(&filepath); + let index_reader = SortedIndexReader::open(&index_path)?; + Ok(Self { + index_reader, + tree_file, + mmap, filepath, block_cache: IndexMap::with_capacity(CACHE_CAPACITY), _marker: PhantomData, @@ -354,13 +372,31 @@ where /// Create from an explicit index path. pub fn with_index_path( filepath: PathBuf, - tree_file: BufReader, + tree_file: Option>, index_path: &Path, ) -> io::Result { let index_reader = SortedIndexReader::open(index_path)?; Ok(Self { index_reader, tree_file, + mmap: None, + filepath, + block_cache: IndexMap::with_capacity(CACHE_CAPACITY), + _marker: PhantomData, + }) + } + + pub fn with_index_path_hybrid( + filepath: PathBuf, + tree_file: Option>, + mmap: Option, + index_path: &Path, + ) -> io::Result { + let index_reader = SortedIndexReader::open(index_path)?; + Ok(Self { + index_reader, + tree_file, + mmap, filepath, block_cache: IndexMap::with_capacity(CACHE_CAPACITY), _marker: PhantomData, @@ -391,50 +427,123 @@ where /// Read a single value directly from the tree file at the given offset. fn read_value_single(&mut self, offset: u64, length: u32) -> io::Result { - self.tree_file.seek(SeekFrom::Start(offset))?; + if let Some(mmap) = &self.mmap { + let mut cursor = io::Cursor::new(mmap.as_ref()); + cursor.set_position(offset); - // Read compression flag - let mut flag = [0u8; 1]; - self.tree_file.read_exact(&mut flag)?; + let mut flag = [0u8; 1]; + cursor.read_exact(&mut flag)?; - let data = if flag[0] == COMPRESSION_FLAG_LZ4 { - // Compressed: [flag:1][lz4_payload_with_prepended_size] - let compressed_len = length as usize - 1; - let mut compressed = vec![0u8; compressed_len]; - self.tree_file.read_exact(&mut compressed)?; + let data = if flag[0] == COMPRESSION_FLAG_LZ4 { + let compressed_len = length as usize - 1; + let mut compressed = vec![0u8; compressed_len]; + cursor.read_exact(&mut compressed)?; + lz4_flex::decompress_size_prepended(&compressed).map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, format!("LZ4 decompression failed: {e}")) + })? + } else { + let payload_len = length as usize - 1; + let mut data = vec![0u8; payload_len]; + cursor.read_exact(&mut data)?; + data + }; + crate::utils::binary_deserialize(&data) + } else if let Some(tree_file) = &mut self.tree_file { + tree_file.seek(SeekFrom::Start(offset))?; - lz4_flex::decompress_size_prepended(&compressed).map_err(|e| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("LZ4 decompression failed: {e}"), - ) - })? + // Read compression flag + let mut flag = [0u8; 1]; + tree_file.read_exact(&mut flag)?; + + let data = if flag[0] == COMPRESSION_FLAG_LZ4 { + // Compressed: [flag:1][lz4_payload_with_prepended_size] + let compressed_len = length as usize - 1; + let mut compressed = vec![0u8; compressed_len]; + tree_file.read_exact(&mut compressed)?; + + lz4_flex::decompress_size_prepended(&compressed).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("LZ4 decompression failed: {e}"), + ) + })? + } else { + // Uncompressed: [flag:1][payload] + let payload_len = length as usize - 1; + let mut data = vec![0u8; payload_len]; + tree_file.read_exact(&mut data)?; + data + }; + + crate::utils::binary_deserialize(&data) } else { - // Uncompressed: [flag:1][payload] - let payload_len = length as usize - 1; - let mut data = vec![0u8; payload_len]; - self.tree_file.read_exact(&mut data)?; - data - }; - - crate::utils::binary_deserialize(&data) + Err(io::Error::other("No data source available")) + } } /// Read a value from a packed block at the given index. fn read_value_packed(&mut self, block_offset: u64, value_index: u16) -> io::Result { + if let Some(mmap) = &self.mmap { + let start = usize::try_from(block_offset).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + if start >= mmap.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Packed block offset out of range")); + } + let end = (start + PAGE_SIZE_USIZE).min(mmap.len()); + let block_buffer = &mmap[start..end]; + if block_buffer.len() < 4 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Packed block truncated")); + } + // Read count (first 4 bytes) + let count = u32::from_le_bytes(block_buffer[0..4].try_into().map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, format!("Invalid count: {e}")) + })?); + + if u32::from(value_index) >= count { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Value index {value_index} out of bounds (count: {count})"), + )); + } + + let mut pos = 4; + // Skip to target value + for i in 0..=value_index { + if pos + 4 > block_buffer.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Packed block corrupted")); + } + let len = u32::from_le_bytes(block_buffer[pos..pos + 4].try_into().map_err(|e| { + io::Error::new(io::ErrorKind::InvalidData, format!("Invalid length: {e}")) + })?) as usize; + pos += 4; + + if i == value_index { + if pos + len > block_buffer.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Packed block corrupted")); + } + let value_data = &block_buffer[pos..pos + len]; + return crate::utils::binary_deserialize(value_data); + } + pos += len; + } + return Err(io::Error::other("Value not found")); + } // Try cache first if !self.block_cache.contains_key(&block_offset) { // Miss - read from disk - self.tree_file.seek(SeekFrom::Start(block_offset))?; - let mut buf = vec![0u8; PAGE_SIZE_USIZE]; - self.tree_file.read_exact(&mut buf)?; + if let Some(tree_file) = &mut self.tree_file { + tree_file.seek(SeekFrom::Start(block_offset))?; + let mut buf = vec![0u8; PAGE_SIZE_USIZE]; + tree_file.read_exact(&mut buf)?; - // Update cache - if self.block_cache.len() >= CACHE_CAPACITY { - self.block_cache.shift_remove_index(0); // FIFO eviction + // Update cache + if self.block_cache.len() >= CACHE_CAPACITY { + self.block_cache.shift_remove_index(0); // FIFO eviction + } + self.block_cache.insert(block_offset, buf); + } else { + return Err(io::Error::other("No data source available")); } - self.block_cache.insert(block_offset, buf); } let block_buffer = self.block_cache.get(&block_offset).unwrap(); @@ -526,19 +635,19 @@ mod tests { // Write entries with value locations let mut writer = SortedIndexWriter::::new(&path).unwrap(); writer.push( - &"apple".to_string(), - &1u32, - ValueLocation::Single { offset: 100, length: 50 } + &"apple".to_string(), + &1u32, + ValueLocation::Single { offset: 100, length: 50 }, ).unwrap(); writer.push( - &"banana".to_string(), - &2u32, - ValueLocation::Packed { block_offset: 200, index: 3, length: 75 } + &"banana".to_string(), + &2u32, + ValueLocation::Packed { block_offset: 200, index: 3, length: 75 }, ).unwrap(); writer.push( - &"cherry".to_string(), - &3u32, - ValueLocation::Single { offset: 300, length: 100 } + &"cherry".to_string(), + &3u32, + ValueLocation::Single { offset: 300, length: 100 }, ).unwrap(); let count = writer.finish().unwrap(); assert_eq!(count, 3); @@ -594,7 +703,7 @@ mod tests { // Note: collect() consumes the reader, so we check is_empty() and remaining first let remaining = reader.remaining; assert_eq!(remaining, 0); - + let entries: Vec<_> = reader.collect(); assert!(entries.is_empty()); } diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 8576b7184..b3c890c34 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -1,9 +1,8 @@ // Import the new MediaQuality struct -use shared::utils::is_blank_optional_string; use crate::model::MediaQuality; use crate::model::{ApiProxyServerInfo, AppConfig, ProxyUserCredentials}; use crate::model::{ConfigTarget, StrmTargetOutput}; -use crate::repository::storage::{ensure_target_storage_path}; +use crate::repository::storage::ensure_target_storage_path; use crate::repository::storage_const; use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, IO_BUFFER_SIZE}; @@ -12,14 +11,16 @@ use filetime::{set_file_times, FileTime}; use log::{error, trace}; use regex::Regex; use serde::Serialize; -use shared::error::{TuliproxError, info_err_res}; -use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType}; -use shared::utils::{ arc_str_serde, extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; +use shared::error::{info_err_res, TuliproxError}; +use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle}; +use shared::utils::{arc_str_option_serde, arc_str_serde, extract_extension_from_url, hash_bytes, + hash_string_as_hex, is_blank_optional_arc_str, truncate_string, ExportStyleConfig, CONSTANTS}; use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::fs::{create_dir_all, remove_dir, remove_file, File}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; +use shared::model::UUIDType; /// Sanitizes a string to be safe for use as a file or directory name by /// following a strict "allow-list" approach and discarding invalid characters. @@ -84,7 +85,7 @@ fn extract_match(name: &str, pattern: &Regex) -> (String, Option) { fn style_rename_year<'a>( name: &'a str, style: &ExportStyleConfig, - release_date: Option<&'a String>, + release_date: Option<&Arc>, ) -> (std::borrow::Cow<'a, str>, Option) { let mut years = Vec::new(); @@ -132,18 +133,20 @@ pub fn strm_get_file_paths(file_prefix: &str, target_path: &Path) -> PathBuf { struct StrmItemInfo { #[serde(with = "arc_str_serde")] group: Arc, - title: String, + #[serde(with = "arc_str_serde")] + title: Arc, item_type: PlaylistItemType, #[serde(default, skip_serializing_if = "Option::is_none")] provider_id: Option, virtual_id: u32, #[serde(with = "arc_str_serde")] input_name: Arc, - url: String, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - series_name: Option, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - release_date: Option, + #[serde(with = "arc_str_serde")] + url: Arc, + #[serde(with = "arc_str_option_serde", skip_serializing_if = "is_blank_optional_arc_str")] + series_name: Option>, + #[serde(with = "arc_str_option_serde", skip_serializing_if = "is_blank_optional_arc_str")] + release_date: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] season: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -209,7 +212,7 @@ fn extract_item_info(pli: &mut PlaylistItem) -> StrmItemInfo { input_name, url, series_name, - release_date: release_date.as_deref().map(ToString::to_string), + release_date, season, episode, added: added.as_ref().map_or_else(|| Some(0), |a| a.parse::().ok()), @@ -753,7 +756,7 @@ pub async fn write_strm_playlist( // create content let url = get_strm_url(target_force_redirect, user_and_server_info.as_ref(), &strm_file.strm_info); let mut content = target_output.strm_props.as_ref().map_or_else(Vec::new, std::clone::Clone::clone); - content.push(url); + content.push(url.to_string()); let content_text = content.join("\r\n"); let content_as_bytes = content_text.as_bytes(); let content_hash = hash_bytes(content_as_bytes); @@ -922,7 +925,7 @@ fn get_strm_url( target_force_redirect: Option<&ClusterFlags>, user_and_server_info: Option<&(ProxyUserCredentials, ApiProxyServerInfo)>, str_item_info: &StrmItemInfo, -) -> String { +) -> Arc { let Some((user, server_info)) = user_and_server_info else { return str_item_info.url.clone(); }; let redirect = user.proxy.is_redirect(str_item_info.item_type) || target_force_redirect.is_some_and(|f| f.has_cluster(str_item_info.item_type)); @@ -940,7 +943,7 @@ fn get_strm_url( | PlaylistItemType::LocalVideo => Some("movie"), _ => None, } { - let url = str_item_info.url.as_str(); + let url = &str_item_info.url; let ext = extract_extension_from_url(url) .map_or_else(String::new, std::string::ToString::to_string); format!( @@ -949,7 +952,7 @@ fn get_strm_url( user.username, user.password, str_item_info.virtual_id - ) + ).into() } else { str_item_info.url.clone() } diff --git a/backend/src/repository/target_id_mapping.rs b/backend/src/repository/target_id_mapping.rs index b5ccd705b..017167213 100644 --- a/backend/src/repository/target_id_mapping.rs +++ b/backend/src/repository/target_id_mapping.rs @@ -1,15 +1,16 @@ +use crate::repository::bplustree::{BPlusTree, BPlusTreeMetadata, BPlusTreeUpdate}; +use chrono::Local; +use log::error; +use serde::{Deserialize, Serialize}; +use shared::error::TuliproxError; +use shared::info_err; +use shared::model::PlaylistItemType; +use shared::model::UUIDType; use std::cmp::max; use std::collections::BTreeMap; use std::io::Error; use std::path::{Path, PathBuf}; -use chrono::Local; -use log::error; -use serde::{Deserialize, Serialize}; - -use shared::model::{PlaylistItemType, UUIDType}; -use crate::repository::bplustree::BPlusTree; - // TODO make configurable const EXPIRATION_DURATION: i64 = 86400; @@ -38,106 +39,307 @@ impl VirtualIdRecord { } } + +/// Helper to get UUID index path from primary path +fn get_uuid_index_path(path: &Path) -> PathBuf { + path.with_extension("uuid.db") +} + +/// Ensure B+tree file exists, creating empty if needed +fn ensure_tree_file(path: &Path) -> std::io::Result<()> +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + if !path.exists() { + BPlusTree::::new().store(path)?; + } + Ok(()) +} + + +/// Dual-mode `TargetIdMapping` supporting both memory-cached and disk-only operations. +/// +/// - `use_memory_cache=true`: Loads UUID index into RAM for O(1) lookups, writes to both memory and disk +/// - `use_memory_cache=false`: Uses disk-based B+tree queries (O(log n)), writes to disk only pub struct TargetIdMapping { - dirty: bool, virtual_id_counter: u32, - by_virtual_id: BPlusTree, - by_uuid: BTreeMap, + use_memory_cache: bool, + // Disk-based handles + disk_by_virtual_id: BPlusTreeUpdate, + disk_by_uuid: BPlusTreeUpdate, + // In-memory cache (only populated when use_memory_cache=true) + mem_by_uuid: Option>, + // Batch buffers for efficient disk writes + pending_virtual_id_upserts: Vec<(u32, VirtualIdRecord)>, + pending_uuid_upserts: Vec<(UUIDType, u32)>, path: PathBuf, } + impl TargetIdMapping { - pub fn new(path: &Path) -> Self { - let tree_virtual_id: BPlusTree = BPlusTree::::load(path).unwrap_or_else(|_| BPlusTree::::new()); - let mut tree_uuid = BTreeMap::new(); + /// Create a new `TargetIdMapping` with dual-mode support. + /// + /// - `use_memory_cache=true`: Full memory mode with O(1) lookups + /// - `use_memory_cache=false`: Disk-only mode with O(log n) lookups + pub fn new(path: &Path, use_memory_cache: bool) -> Result { + let uuid_index_path = get_uuid_index_path(path); + + let uuid_index_existed = uuid_index_path.exists(); + + // Ensure both tree files exist + ensure_tree_file::(path) + .map_err(|e| info_err!("Failed to create primary tree at {}: {e}", path.display()))?; + ensure_tree_file::(&uuid_index_path) + .map_err(|e| info_err!("Failed to create UUID index at {}: {e}", uuid_index_path.display()))?; + + // Open disk-based update handles + let mut disk_by_virtual_id = match BPlusTreeUpdate::::try_new(path) { + Ok(tree) => tree, + Err(e) => { + error!("Failed to open primary tree at {}: {e}", path.display()); + // Create fresh and try again + let _ = BPlusTree::::new().store(path); + BPlusTreeUpdate::try_new(path).map_err(|_| info_err!("Failed to create primary tree after retry"))? + } + }; + + let mut disk_by_uuid = match BPlusTreeUpdate::::try_new(&uuid_index_path) { + Ok(tree) => tree, + Err(e) => { + error!("Failed to open UUID index at {}: {e}", uuid_index_path.display()); + // Create fresh and try again + let _ = BPlusTree::::new().store(&uuid_index_path); + BPlusTreeUpdate::try_new(&uuid_index_path).map_err(|_| info_err!("Failed to create UUID index after retry"))? + } + }; + let mut virtual_id_counter: u32 = 0; - tree_virtual_id.traverse(|keys, values| { - match keys.iter().max() { - None => {} - Some(max_value) => { - virtual_id_counter = max(virtual_id_counter, *max_value); - } - } - for v in values { - tree_uuid.insert(v.uuid, v.virtual_id); - } - }); - Self { - dirty: false, - virtual_id_counter, - by_virtual_id: tree_virtual_id, - by_uuid: tree_uuid, - path: path.to_path_buf(), + let mut meta_found = false; + + // Try load from header metadata (using disk handle to avoid full tree load) + if let Ok(BPlusTreeMetadata::TargetIdMapping(val)) = disk_by_virtual_id.get_metadata() { + virtual_id_counter = val; + meta_found = true; } + + let mut uuid_index_entries: Vec<(UUIDType, u32)> = Vec::new(); + let needs_uuid_rebuild = !uuid_index_existed; + + // Determines if we MUST traverse the tree + // We traverse if: + // 1. We need to populate memory cache (use_memory_cache) + // 2. We need to rebuild UUID index (needs_uuid_rebuild) + // 3. We didn't find metadata (so we need to find max id manually) + let needs_traversal = use_memory_cache || needs_uuid_rebuild || !meta_found; + + let mem_by_uuid = if needs_traversal { + // Load primary tree only if traversal is needed + let tree: BPlusTree = BPlusTree::load(path) + .map_err(|e| { + error!("Failed to load primary tree at {}, starting fresh: {e}", path.display()); + e + }) + .unwrap_or_else(|_| BPlusTree::new()); + + // Traverse the primary tree to get UUID mappings and max ID + let mut uuid_map = BTreeMap::new(); + tree.traverse(|keys, values| { + if let Some(max_key) = keys.iter().max() { + virtual_id_counter = max(virtual_id_counter, *max_key); + } + for v in values { + uuid_map.insert(v.uuid, v.virtual_id); + if needs_uuid_rebuild { + uuid_index_entries.push((v.uuid, v.virtual_id)); + } + } + }); + + if use_memory_cache { + Some(uuid_map) + } else { + None + } + } else { + // Fast path! No traversal needed + None + }; + + // Rebuild UUID index if it was missing + if needs_uuid_rebuild && !uuid_index_entries.is_empty() { + let batch: Vec<(&UUIDType, &u32)> = uuid_index_entries + .iter() + .map(|(k, v)| (k, v)) + .collect(); + if let Err(e) = disk_by_uuid.upsert_batch(&batch) { + error!("Failed to rebuild UUID index: {e}"); + } + } + + Ok(Self { + virtual_id_counter, + use_memory_cache, + disk_by_virtual_id, + disk_by_uuid, + mem_by_uuid, + pending_virtual_id_upserts: Vec::new(), + pending_uuid_upserts: Vec::new(), + path: path.to_path_buf(), + }) } - // pub fn get_virtual_id(&mut self, uuid: UUIDType, provider_id: u32, item_type: PlaylistItemType, parent_virtual_id: u32) -> u32 { - // match self.by_uuid.get(&uuid) { - // None => { - // self.dirty = true; - // self.virtual_id_counter += 1; - // let record = VirtualIdRecord::new(provider_id, self.virtual_id_counter, item_type, parent_virtual_id, uuid); - // self.by_virtual_id.insert(self.virtual_id_counter, record); - // self.virtual_id_counter - // } - // Some(virtual_id) => *virtual_id - // } - // } - pub fn get_and_update_virtual_id(&mut self, uuid: &UUIDType, provider_id: u32, item_type: PlaylistItemType, parent_virtual_id: u32) -> u32 { - match self.by_uuid.get(uuid) { + + // Lookup existing virtual_id + let existing_virtual_id = if self.use_memory_cache { + self.mem_by_uuid.as_ref().and_then(|m| m.get(uuid).copied()) + } else { + self.disk_by_uuid.query(uuid).ok().flatten() + }; + + match existing_virtual_id { None => { - self.dirty = true; + // New entry: allocate new virtual_id self.virtual_id_counter += 1; let virtual_id = self.virtual_id_counter; let record = VirtualIdRecord::new(provider_id, virtual_id, item_type, parent_virtual_id, *uuid); - self.by_virtual_id.insert(virtual_id, record); - self.virtual_id_counter + + // Buffer for disk write + self.pending_virtual_id_upserts.push((virtual_id, record)); + self.pending_uuid_upserts.push((*uuid, virtual_id)); + + // Update memory cache if enabled + if let Some(ref mut mem_map) = self.mem_by_uuid { + mem_map.insert(*uuid, virtual_id); + } + + virtual_id } Some(virtual_id) => { - if let Some(record) = self.by_virtual_id.query(virtual_id) { - if record.provider_id == provider_id && (record.item_type != item_type || record.parent_virtual_id != parent_virtual_id) { - let new_record = VirtualIdRecord::new(provider_id, *virtual_id, item_type, parent_virtual_id, *uuid); - self.by_virtual_id.insert(*virtual_id, new_record); - self.dirty = true; + // Existing entry: check if update needed + // For update checks, we use the primary tree + let needs_update = match self.disk_by_virtual_id.query(&virtual_id) { + Ok(Some(record)) => { + record.provider_id == provider_id && + (record.item_type != item_type || record.parent_virtual_id != parent_virtual_id) } + Ok(None) => false, + Err(e) => { + error!("Failed to query record for virtual_id {virtual_id}: {e}"); + false + } + }; + + if needs_update { + let new_record = VirtualIdRecord::new(provider_id, virtual_id, item_type, parent_virtual_id, *uuid); + self.pending_virtual_id_upserts.push((virtual_id, new_record)); } - *virtual_id + + virtual_id } } } pub fn persist(&mut self) -> Result<(), Error> { - if self.dirty { - self.by_virtual_id.store(&self.path)?; + if self.has_pending_changes() { + // Flush pending virtual_id upserts + if !self.pending_virtual_id_upserts.is_empty() { + let batch: Vec<(&u32, &VirtualIdRecord)> = self.pending_virtual_id_upserts + .iter() + .map(|(k, v)| (k, v)) + .collect(); + self.disk_by_virtual_id.upsert_batch(&batch)?; + self.pending_virtual_id_upserts.clear(); + } + + // Flush pending UUID index upserts + if !self.pending_uuid_upserts.is_empty() { + let batch: Vec<(&UUIDType, &u32)> = self.pending_uuid_upserts + .iter() + .map(|(k, v)| (k, v)) + .collect(); + self.disk_by_uuid.upsert_batch(&batch)?; + self.pending_uuid_upserts.clear(); + } + + // Persist virtual_id_counter via B+Tree header metadata + self.disk_by_virtual_id + .set_metadata(&BPlusTreeMetadata::TargetIdMapping(self.virtual_id_counter)) + .map_err(|e| { + error!("Failed to write virtual_id_counter to tree header at {}: {e}", self.path.display()); + e + })?; } - self.dirty = false; Ok(()) } + + /// Check if there are pending changes + pub fn has_pending_changes(&self) -> bool { + !self.pending_virtual_id_upserts.is_empty() || !self.pending_uuid_upserts.is_empty() + } } impl Drop for TargetIdMapping { fn drop(&mut self) { - if let Err(err) = self.persist() { - error!("Failed to persist target id mapping {} err:{err}", &self.path.display()); + if self.has_pending_changes() { + if let Err(err) = self.persist() { + error!("Failed to persist target id mapping {} err:{err}", &self.path.display()); + } } } } #[cfg(test)] mod tests { - // use std::path::PathBuf; - // use crate::repository::bplustree::BPlusTree; - // use crate::repository::target_id_mapping::{VirtualIdRecord}; + use super::*; + use shared::model::PlaylistItemType; + use tempfile::tempdir; - // #[test] - // fn test_id_mapping() { - // let path = PathBuf::from("../m3u-test/settings/m3u-silver/data/xt_m3u/id_mapping.db"); - // let mapping = BPlusTree::::load(&path); - // mapping.unwrap().traverse(|keys, values| { - // for (key, value) in keys.iter().zip(values.iter()) { - // println!("{key:?} {value:?}\n"); - // } - // }); - // } + #[test] + fn test_disk_only_mode() -> Result<(), TuliproxError> { + let dir = tempdir().map_err(|_| info_err!("Failed to create temp dir"))?; + let path = dir.path().join("id_mapping.db"); + + // Create mapping in disk-only mode + let uuid1 = UUIDType::default(); + { + let mut mapping = TargetIdMapping::new(&path, false)?; + let vid1 = mapping.get_and_update_virtual_id(&uuid1, 100, PlaylistItemType::Live, 0); + assert_eq!(vid1, 1); + mapping.persist().map_err(|_| info_err!("Failed to persist mapping"))?; + } + + // Reopen and verify persistence + { + let mut mapping = TargetIdMapping::new(&path, false)?; + let vid1_again = mapping.get_and_update_virtual_id(&uuid1, 100, PlaylistItemType::Live, 0); + assert_eq!(vid1_again, 1); // Should get same virtual_id + } + + Ok(()) + } + + #[test] + fn test_memory_cache_mode() -> Result<(), TuliproxError> { + let dir = tempdir().map_err(|_| info_err!("Failed to create temp dir"))?; + let path = dir.path().join("id_mapping_mem.db"); + + let uuid1 = UUIDType::default(); + { + let mut mapping = TargetIdMapping::new(&path, true)?; + let vid1 = mapping.get_and_update_virtual_id(&uuid1, 100, PlaylistItemType::Video, 0); + assert_eq!(vid1, 1); + mapping.persist().map_err(|err| info_err!("{err}"))?; + } + + // Reopen with memory cache and verify + { + let mut mapping = TargetIdMapping::new(&path, true)?; + let vid1_again = mapping.get_and_update_virtual_id(&uuid1, 100, PlaylistItemType::Video, 0); + assert_eq!(vid1_again, 1); + } + + Ok(()) + } } diff --git a/backend/src/repository/user_repository.rs b/backend/src/repository/user_repository.rs index 8c746a0aa..4df00cea8 100644 --- a/backend/src/repository/user_repository.rs +++ b/backend/src/repository/user_repository.rs @@ -259,13 +259,16 @@ async fn save_xtream_user_bouquet_for_target(config: &Config, target_name: &str, if let Some(bouquet_categories) = bouquet { if let Some(xtream_categories) = xtream_get_playlist_categories(config, target_name, cluster).await { let filtered: Vec = xtream_categories.iter().filter(|p| bouquet_categories.contains(&p.name)).cloned().collect(); + if filtered.is_empty() { + if tokio::fs::try_exists(&bouquet_path).await.is_ok_and(|r| r) { + tokio::fs::remove_file(bouquet_path).await?; + } + } else { json_write_documents_to_file(&bouquet_path, &filtered).await.map_err(|err| Error::other(format!("Failed to write xtream bouquet file: {err}")))?; + } } } - if bouquet_path.exists() { - tokio::fs::remove_file(bouquet_path).await?; - } Ok(()) } @@ -381,6 +384,8 @@ pub(crate) async fn user_get_series_bouquet(cfg: &Config, username: &str, target /// # Returns /// * `Some(HashSet)` - Set of category IDs or names to include /// * `None` - No filtering applied +/// +/// TODO xtream converts ids to u32 again, separate m3u and xtream handling pub async fn user_get_bouquet_filter(config: &Config, username: &str, category_id: Option, target: TargetType, cluster: XtreamCluster) -> Option> { if let Some(cid) = category_id { if cid > 0 { @@ -402,7 +407,7 @@ pub async fn user_get_bouquet_filter(config: &Config, username: &str, category_i // xtream filter has PlaylistXtreamCategory serde_json::from_str::>(&bouquet_categories) .ok() - .map(|v| v.into_iter().map(|c| c.id).collect()) + .map(|v| v.into_iter().map(|c| c.id.to_string()).collect()) } else { // m3u filter has only group names serde_json::from_str::>(&bouquet_categories).ok() diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index a5fe6e674..3d53344d3 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -1,10 +1,10 @@ use crate::api::model::AppState; -use crate::model::PlaylistXtreamCategory; +use crate::model::{ConfigInput, PlaylistXtreamCategory}; use crate::model::{AppConfig, ProxyUserCredentials}; use crate::model::{Config, ConfigTarget}; use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate}; use crate::repository::playlist_scratch::PlaylistScratch; -use crate::repository::storage::{get_file_path_for_db_index, get_target_id_mapping_file, get_target_storage_path}; +use crate::repository::storage::{get_input_storage_path, get_target_id_mapping_file, get_target_storage_path}; use crate::repository::storage_const; use crate::repository::target_id_mapping::VirtualIdRecord; use crate::repository::xtream_playlist_iterator::XtreamPlaylistJsonIterator; @@ -17,10 +17,10 @@ use indexmap::IndexMap; use log::error; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use shared::error::{info_err_res, info_err, notify_err, string_to_io_error, TuliproxError}; +use shared::error::{info_err_res, notify_err, string_to_io_error, TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemType, SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}; -use shared::utils::{arc_str_serde, get_u32_from_serde_value, intern, StringInterner}; +use shared::utils::{arc_str_serde, get_u32_from_serde_value, Internable}; use std::collections::HashMap; use std::fs::File; use std::io::{Error, ErrorKind}; @@ -174,7 +174,7 @@ pub type CategoryKey = (XtreamCluster, Arc); // Because interner is not thread safe we can't use it currently for interning. // We leave the argument for later optimizations. -async fn load_old_category_ids(path: &Path, _interner: &mut StringInterner) -> (u32, HashMap) { +async fn load_old_category_ids(path: &Path) -> (u32, HashMap) { let old_path = path.to_path_buf(); tokio::task::spawn_blocking(move || { let mut result: HashMap = HashMap::new(); @@ -195,7 +195,7 @@ async fn load_old_category_ids(path: &Path, _interner: &mut StringInterner) -> ( if let Some(category_id) = entry.get(crate::model::XC_TAG_CATEGORY_ID).and_then(get_u32_from_serde_value) { if let Value::Object(item) = entry { if let Some(category_name) = get_map_item_as_str(&item, crate::model::XC_TAG_CATEGORY_NAME) { - result.insert((cluster, /*interner.*/intern(&category_name)), category_id); + result.insert((cluster, /*interner.*/category_name.intern()), category_id); max_id = max_id.max(category_id); } } @@ -242,7 +242,6 @@ pub async fn xtream_write_playlist( app_cfg: &Arc, target: &ConfigTarget, playlist: &mut [PlaylistGroup], - interner: &mut StringInterner, ) -> Result<(), TuliproxError> { let path = { let config = app_cfg.config.load(); @@ -256,7 +255,7 @@ pub async fn xtream_write_playlist( let mut series_col = Vec::with_capacity(50_000); let mut vod_col = Vec::with_capacity(50_000); - let categories = create_categories(playlist, &path, interner).await; + let categories = create_categories(playlist, &path).await; { for (xtream_cluster, category) in categories { match xtream_cluster { @@ -320,15 +319,15 @@ pub async fn xtream_write_playlist( Ok(()) } -async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path, interner: &mut StringInterner) -> Vec<(XtreamCluster, CategoryEntry)> { +async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<(XtreamCluster, CategoryEntry)> { // preserve category_ids - let (max_cat_id, existing_cat_ids) = load_old_category_ids(path, interner).await; + let (max_cat_id, existing_cat_ids) = load_old_category_ids(path).await; let mut cat_id_counter = max_cat_id; let mut new_categories: IndexMap = IndexMap::new(); let mut last_cluster: Option = None; - let mut last_group = intern(""); + let mut last_group = "".intern(); let mut last_category_id: u32 = 0; for plg in playlist.iter_mut() { @@ -578,45 +577,50 @@ pub async fn xtream_load_rewrite_playlist( XtreamPlaylistJsonIterator::new(cluster, config, target, category_id, user).await } -pub async fn iter_raw_xtream_playlist(app_config: &AppConfig, target: &ConfigTarget, cluster: XtreamCluster) -> Option<(FileReadGuard, impl Iterator)> { +pub async fn iter_raw_xtream_target_playlist(app_config: &AppConfig, target: &ConfigTarget, cluster: XtreamCluster) -> Option<(FileReadGuard, Box + Send>)> { let config = app_config.config.load(); - if let Some(storage_path) = xtream_get_storage_path(&config, target.name.as_str()) { - let xtream_path = xtream_get_file_path(&storage_path, cluster); - if !xtream_path.exists() { - return None; - } - let file_lock = app_config.file_locks.read_lock(&xtream_path).await; - match BPlusTreeQuery::::try_new(&xtream_path) - .map_err(|err| info_err!("Could not open BPlusTreeQuery {xtream_path:?} - {err}")) { - Ok(mut query) => { - let index_path = get_file_path_for_db_index(&xtream_path); - let items: Vec = if index_path.exists() { - match query.disk_iter_sorted::() { - Ok(iter) => iter.filter_map(Result::ok).map(|(_, v)| v).collect(), - Err(err) => { - error!("Sorted index error {}: {err}", xtream_path.display()); - // Re-open query for fallback - match BPlusTreeQuery::::try_new(&xtream_path) { - Ok(mut query) => query.iter().map(|(_, v)| v).collect(), - Err(fallback_err) => { - error!("Fallback query also failed {}: {fallback_err}", xtream_path.display()); - Vec::new() - } - } + let storage_path = xtream_get_storage_path(&config, target.name.as_str())?; + let xtream_path = xtream_get_file_path(&storage_path, cluster); + iter_raw_xtream_playlist::(app_config, &xtream_path).await +} + +pub async fn iter_raw_xtream_input_playlist(app_config: &AppConfig, input: &ConfigInput, cluster: XtreamCluster) -> Option<(FileReadGuard, Box + Send>)> { + let config = app_config.config.load(); + let working_dir = &config.working_dir; + let storage_path = get_input_storage_path(&input.name, working_dir).ok()?; + let xtream_path = xtream_get_file_path(&storage_path, cluster); + + iter_raw_xtream_playlist::(app_config, &xtream_path).await +} + +async fn iter_raw_xtream_playlist(app_config: &AppConfig, xtream_path: &Path) -> Option<(FileReadGuard, Box + Send>)> +where + ItemKey: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Send + 'static, + SortKey: for<'de> Deserialize<'de> + Send + 'static, +{ + let file_lock = app_config.file_locks.read_lock(xtream_path).await; + if !tokio::fs::try_exists(xtream_path).await.unwrap_or(false) { + return None; + } + + let iter: Box + Send> = { + match BPlusTreeQuery::::try_new(xtream_path) { + Ok(tree) => match tree.disk_iter_sorted::() { + Ok(sorted_iter) => Box::new(sorted_iter.filter_map(Result::ok).map(|(_, v)| v)), + Err(_) => { + match BPlusTreeQuery::::try_new(xtream_path) { + Ok(tree) => Box::new(tree.disk_iter().map(|(_, v)| v)), + Err(_) => return None, } } - } else { - query.iter().map(|(_, v)| v).collect() - }; + } + Err(_) => { + return None + } + } + }; - let len = items.len(); - Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i + 1 < len)))) - } - Err(_) => None - } - } else { - None - } + Some((file_lock, iter)) } pub fn playlist_iter_to_stream(channels: Option<(FileReadGuard, I)>) -> impl Stream> @@ -670,7 +674,7 @@ pub async fn persist_input_xtream_playlist(app_config: &Arc, storage_ // load for cluster in XTREAM_CLUSTER { let xtream_path = xtream_get_file_path(storage_path, cluster); - if let Ok(true) = tokio::fs::try_exists(&xtream_path).await { + if tokio::fs::try_exists(&xtream_path).await.unwrap_or(false) { let file_lock = app_config.file_locks.read_lock(&xtream_path).await; let stored_entries = stored_scratch.get_mut(cluster); if let Ok(mut query) = BPlusTreeQuery::::try_new(&xtream_path) { @@ -741,7 +745,7 @@ pub async fn persist_input_xtream_playlist(app_config: &Arc, storage_ }; let data = fetched_categories.get_mut(cluster); // if there is no data save only if no file exists! Prevent data loss from failed download attempt - if !data.is_empty() || tokio::fs::try_exists(&col_path).await.is_ok_and(|v| !v) { + if !data.is_empty() || !tokio::fs::try_exists(&col_path).await.unwrap_or(false) { let lock = app_cfg.file_locks.write_lock(&col_path).await; if let Err(err) = json_write_documents_to_file(&col_path, data).await { errors.push(format!("Persisting collection failed: {}: {err}", col_path.display())); @@ -930,7 +934,7 @@ pub async fn load_input_xtream_playlist(app_config: &Arc, storage_pat groups.entry((cluster, cat_id)) .or_insert_with(|| PlaylistGroup { id: cat_id, - title: intern("Unknown"), + title: "Unknown".intern(), channels: Vec::new(), xtream_cluster: cluster, }) diff --git a/backend/src/utils/file/csv_input_reader.rs b/backend/src/utils/file/csv_input_reader.rs index 669ee1342..c23c3c168 100644 --- a/backend/src/utils/file/csv_input_reader.rs +++ b/backend/src/utils/file/csv_input_reader.rs @@ -6,8 +6,9 @@ use std::io; use std::io::{BufRead, Cursor, Error}; use std::path::PathBuf; use url::Url; +use shared::concat_string; use shared::model::{ConfigInputAliasDto, InputType}; -use shared::utils::{get_credentials_from_url, parse_timestamp, trim_last_slash}; +use shared::utils::{get_credentials_from_url, parse_timestamp, trim_last_slash, Internable}; use crate::utils::request::get_local_file_content; const CSV_SEPARATOR: char = ';'; @@ -55,9 +56,9 @@ fn csv_assign_mandatory_fields(alias: &mut ConfigInputAliasDto, input_type: Inpu let username = alias.username.as_deref().unwrap_or_default(); let domain: Vec<&str> = url.domain().unwrap_or_default().split('.').collect(); if domain.len() > 1 { - alias.name = format!("{}_{username}", domain[domain.len() - 2]); + alias.name = concat_string!(domain[domain.len() - 2], "_", username).intern(); } else { - alias.name = username.to_string(); + alias.name = username.intern(); } } } @@ -83,7 +84,7 @@ fn csv_assign_config_input_column(config_input: &mut ConfigInputAliasDto, header config_input.priority = priority; } FIELD_NAME => { - config_input.name = value.to_string(); + config_input.name = value.intern(); } FIELD_USERNAME => { config_input.username = Some(value.to_string()); @@ -142,7 +143,7 @@ pub fn csv_read_inputs_from_reader(batch_input_type: InputType, reader: impl Buf let mut config_input = ConfigInputAliasDto { id: 0, - name: String::new(), + name: "".intern(), url: String::new(), username: None, password: None, diff --git a/backend/src/utils/file/file_lock_manager.rs b/backend/src/utils/file/file_lock_manager.rs index e29a3238e..7bdd74f5b 100644 --- a/backend/src/utils/file/file_lock_manager.rs +++ b/backend/src/utils/file/file_lock_manager.rs @@ -79,6 +79,23 @@ impl FileLockManager { FileWriteGuard::new(guard) } + /// Tries to acquire a write lock using a raw string key instead of a normalized `Path`. + /// + /// Unlike the standard path-based locks, this method does **not** perform any + /// path normalization or conversion. The string is used directly as the lock key, + /// which can be useful for non-file-based identifiers or dynamic keys. + /// + /// Returns immediately with an error if the lock is currently held, rather than + /// waiting for it to become available. + pub async fn try_write_lock_str(&self, text: &str) -> io::Result { + let lock_key = LockKey::Str(text.to_string()); + let file_lock = self.get_or_create_lock(lock_key).await; + match Arc::clone(&file_lock).try_write_owned() { + Ok(lock_guard) => Ok(FileWriteGuard::new(lock_guard)), + Err(_) => Err(str_to_io_error("Failed to acquire write lock")) + } + } + fn get_lock_key_for_path(path: &Path) -> LockKey { let normalized_path = normalize_path(path); diff --git a/backend/src/utils/mod.rs b/backend/src/utils/mod.rs index a4c664bcc..f1f8a9ee5 100644 --- a/backend/src/utils/mod.rs +++ b/backend/src/utils/mod.rs @@ -18,6 +18,7 @@ pub use self::trakt::*; pub use self::telegram::*; pub use self::geoip::*; pub use self::db_viewer::*; +pub use shared::utils::*; #[macro_export] macro_rules! debug_if_enabled { diff --git a/backend/src/utils/network/epg.rs b/backend/src/utils/network/epg.rs index 62d1106c6..21edc47b0 100644 --- a/backend/src/utils/network/epg.rs +++ b/backend/src/utils/network/epg.rs @@ -6,6 +6,7 @@ use log::debug; use shared::error::{TuliproxError, info_err}; use shared::utils::{sanitize_sensitive_info, short_hash}; use std::path::PathBuf; +use shared::concat_string; use crate::processing::processor::playlist::PlaylistProcessingContext; use crate::repository::storage::get_input_storage_path; use crate::repository::storage_const; @@ -17,7 +18,7 @@ pub fn get_input_raw_epg_file_path(url: &str, input: &ConfigInput, working_dir: if let Some(persist_path) = input.persist.as_deref() { if !persist_path.is_empty() { if let Some(path) = prepare_file_path(input.persist.as_deref(), working_dir, "") - .map(|path| add_prefix_to_filename(&path, format!("{file_prefix}_epg_").as_str(), Some("xml"))) { + .map(|path| add_prefix_to_filename(&path, concat_string!(&file_prefix, "_epg_").as_str(), Some("xml"))) { return Ok(path); } } @@ -45,7 +46,7 @@ async fn download_epg_file(url: &str, ctx: &PlaylistProcessingContext, input: &C // Cache miss: file doesn't exist } - let lock_key = persist_file_path.display().to_string(); + let lock_key: std::sync::Arc = persist_file_path.display().to_string().into(); let _input_lock = ctx.get_input_lock(&lock_key).await; if ctx.is_input_downloaded(&lock_key).await { diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 91bf5eb02..f88b14307 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -637,6 +637,12 @@ pub fn parse_range(range: &str) -> Option<(u64, Option)> { Some((start, end)) } +pub fn is_file_url(url: &str) -> bool { + Url::parse(url) + .map(|u| u.scheme().eq_ignore_ascii_case("file")) + .unwrap_or(false) +} + #[cfg(test)] mod tests { use shared::utils::{get_base_url_from_str, replace_url_extension, sanitize_sensitive_info}; diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index 6c2f18886..75cfbec71 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -13,9 +13,9 @@ use crate::repository::xtream_repository::{persist_input_vod_info, persists_inpu use crate::utils::request; use chrono::{DateTime, Utc}; use log::{error, info, warn}; -use shared::error::{string_to_io_error, to_io_error, TuliproxError}; +use shared::error::{TuliproxError}; use shared::model::{MsgKind, PlaylistEntry, PlaylistGroup, ProxyUserStatus, SeriesStreamProperties, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamSeriesInfo, XtreamVideoInfo, XtreamVideoInfoDoc}; -use shared::utils::{extract_extension_from_url, get_i64_from_serde_value, get_string_from_serde_value, sanitize_sensitive_info, StringInterner}; +use shared::utils::{extract_extension_from_url, get_i64_from_serde_value, get_string_from_serde_value, sanitize_sensitive_info, Internable}; use std::collections::HashMap; use std::io::Error; use std::path::Path; @@ -26,7 +26,8 @@ use crate::utils::request::DynReader; use fs2::FileExt; use std::fs::File; use std::sync::Arc; -use shared::{notify_err, notify_err_res}; + +use shared::{info_err, notify_err, notify_err_res}; const THREE_DAYS_IN_SECS: i64 = 3 * 24 * 60 * 60; @@ -74,22 +75,22 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, target: &ConfigTarget, pli: &XtreamPlaylistItem, info_url: &str, - cluster: XtreamCluster) -> Result { - let xtream_output = target.get_xtream_output().ok_or_else(|| Error::other("Unexpected error, missing xtream output"))?; + cluster: XtreamCluster) -> Result { + let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!("Unexpected error, missing xtream output"))?; let app_config = &app_state.app_config; let server_info = app_config.get_user_server_info(user); let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str())); if let Some(content) = pli.get_resolved_info_document(&options) { - return serde_json::to_string(&content).map_err(to_io_error); + return serde_json::to_string(&content).map_err(|err| info_err!("{err}")); } let input_source = InputSource::from(input).with_url(info_url.to_owned()); if let Ok(content) = get_xtream_stream_info_content(client, &input_source, false).await { if content.is_empty() { - return Err(string_to_io_error(format!("Provider returned no response for stream with id: {}/{}/{}", - target.name.replace(' ', "_").as_str(), &cluster, pli.get_virtual_id()))); + return Err(info_err!("Provider returned no response for stream with id: {}/{}/{}", + target.name.replace(' ', "_").as_str(), &cluster, pli.get_virtual_id())); } if let Some(provider_id) = pli.get_provider_id() { match cluster { @@ -136,14 +137,14 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, Ok(info) => { // parse series info let series_stream_props = SeriesStreamProperties::from_info(&info, pli); + if let Ok(storage_path) = get_input_storage_path(&input.name, working_dir) { // update input db if let Err(err) = persists_input_series_info(app_config, &storage_path, cluster, &input.name, provider_id, &series_stream_props).await { error!("Failed to persist series info for input {}: {err}", &input.name); } } - let mut interner = StringInterner::new(); - if let Some(mut episodes) = parse_xtream_series_info(&pli.get_uuid(), &series_stream_props, &group, &series_name, input, &mut interner) { + if let Some(mut episodes) = parse_xtream_series_info(&pli.get_uuid(), &series_stream_props, &group, &series_name, input) { let config = &app_state.app_config.config.load(); match get_target_storage_path(config, target.name.as_str()) { None => { @@ -151,16 +152,17 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, } Some(target_path) => { let mut in_memory_updates = Vec::new(); - let mut provider_series: HashMap> = HashMap::new(); + let mut provider_series: HashMap, Vec> = HashMap::new(); { - let (mut target_id_mapping, _file_lock) = get_target_id_mapping(&app_state.app_config, &target_path).await; + let (mut target_id_mapping, _file_lock) = get_target_id_mapping(&app_state.app_config, &target_path, target.use_memory_cache).await?; + if let Some(parent_id) = pli.get_provider_id() { let category_id = pli.get_category_id().unwrap_or(0); for episode in &mut episodes { episode.header.virtual_id = target_id_mapping.get_and_update_virtual_id(&episode.header.uuid, provider_id, episode.header.item_type, parent_id); episode.header.category_id = category_id; let episode_provider_id = episode.header.get_provider_id().unwrap_or(0); - provider_series.entry(pli.get_uuid().to_string()) + provider_series.entry(pli.get_uuid().intern()) .or_default() .push(ProviderEpisodeKey { provider_id: episode_provider_id, @@ -224,18 +226,18 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, } } - Err(string_to_io_error(format!("Can't find stream with id: {}/{}/{}", - target.name.replace(' ', "_").as_str(), &cluster, pli.get_virtual_id()))) + Err(info_err!("Can't find stream with id: {}/{}/{}", + target.name.replace(' ', "_").as_str(), &cluster, pli.get_virtual_id())) } fn xtream_resolve_stream_info(app_state: &Arc, user: &ProxyUserCredentials, target: &ConfigTarget, xtream_output: &XtreamTargetOutput, - pli: &XtreamPlaylistItem) -> Option> { + pli: &XtreamPlaylistItem) -> Option> { let app_config = &app_state.app_config; let server_info = app_config.get_user_server_info(user); let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str())); if let Some(content) = pli.get_resolved_info_document(&options) { - return Some(serde_json::to_string(&content).map_err(to_io_error)); + return Some(serde_json::to_string(&content).map_err(|err| info_err!("Failed to serialize stream info: {err}"))); } None } @@ -487,10 +489,10 @@ pub fn create_vod_info_from_item(target: &ConfigTarget, user: &ProxyUserCredenti doc.info.name.clone_from(name); doc.movie_data.stream_id = stream_id; doc.movie_data.name.clone_from(name); - doc.movie_data.added = added.to_string(); - doc.movie_data.category_id = category_id.to_string(); + doc.movie_data.added = added.intern(); + doc.movie_data.category_id = category_id.intern(); doc.movie_data.category_ids.push(category_id); - doc.movie_data.container_extension = extension; + doc.movie_data.container_extension = extension.intern(); doc.movie_data.custom_sid = None; serde_json::to_string(&doc).unwrap_or(String::new()) @@ -631,7 +633,7 @@ async fn save_xtream_categories_to_file(col_path: &Path, categories: &[XtreamCat let col_path_buf = col_path.to_path_buf(); let cat_entries: Vec = categories.iter().map(|c| CategoryEntry { category_id: c.category_id, - category_name: shared::utils::intern(&c.category_name), + category_name: c.category_name.clone(), parent_id: 0, }).collect(); diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index f2a2ce391..977ec843b 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "frontend" -version = "3.2.42" +version = "3.2.43" edition = "2021" rust-version = "1.89.0" [dependencies] -shared = { version = "3.2.42", path = "../shared" } +shared = { version = "3.2.43", path = "../shared" } chrono = "0" yew = "0.21" yew-router = "0.18" @@ -33,6 +33,7 @@ regex = "1.12.2" base64 = "0.22.1" cron = "0.15" fastrand = "2.3.0" +indexmap = "2.12.1" [dependencies.web-sys] version = "0.3" diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 3b5ee280d..d5e50e9f2 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -58,7 +58,6 @@ "BUFFER_SIZE": "Buffer Size", "SHARED_BURST_BUFFER_BYTES": "Shared burst buffer (MB)", "ENABLED": "Enabled", - "DISABLED": "Disabled", "SIZE": "Size", "CACHE_DIR": "Cache dir", "NOTIFY_ON": "Notify on", @@ -75,6 +74,8 @@ "DIRECTORY": "Directory", "ORGANIZE_INTO_DIRECTORIES": "Organize Into Directories", "EPISODE_PATTERN": "Episode Pattern", + "API": "Api", + "MAIN": "Main", "MAIN_CONFIG": "Main", "API_CONFIG": "API", "LOG_CONFIG": "Log", @@ -195,6 +196,7 @@ "EDIT": "Edit", "DELETE": "Delete", "ACTIVE": "Active", + "DISABLED": "Disabled", "IGNORE_LOGO": "Ignore logo", "SHARE_LIVE_STREAMS": "Share live", "REMOVE_DUPLICATES": "Remove duplicates", @@ -387,7 +389,10 @@ "BODY_TEMPLATE": "Body Template", "WEBHOOK_URL": "Webhook URL", "AUTO": "Auto", - "API_USER_AGENT": "User Agent" + "API_USER_AGENT": "User Agent", + "ADVANCED": "Advanced", + "TRAKT": "Trakt", + "SEASON": "Season" }, "TITLE": { "USER_BOUQUET_EDITOR": "Playlist Category Selection" diff --git a/frontend/public/assets/icons.json b/frontend/public/assets/icons.json index e6a0a7a66..63b38cbbc 100644 --- a/frontend/public/assets/icons.json +++ b/frontend/public/assets/icons.json @@ -533,5 +533,25 @@ { "keys": [ "VideoLibrary" ], "path": "M4 6H2v14c0 1.1.9 2 2 2h14v-2H4V6zm16-4H8c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h12c1.1 0 2-.9 2-2V4c0-1.1-.9-2-2-2zm0 14H8V4h12v12zM12 5.5v9l6-4.5z" + }, + { + "keys": [ "Options" ], + "path": "M 11.999906,2 C 6.469904,2 2,6.4699087 2,11.999907 c 0,5.53 4.469904,9.999905 9.999906,9.999905 5.530001,0 9.999907,-4.469905 9.999907,-9.999905 C 21.999813,6.4699087 17.529907,2 11.999906,2 Z m 0,1.999878 c 4.410002,0 8.000029,3.5900297 8.000029,8.000029 0,4.409998 -3.590027,8.000028 -8.000029,8.000028 -4.409998,0 -8.0000282,-3.59003 -8.0000282,-8.000028 0,-4.4099993 3.5900302,-8.000029 8.0000282,-8.000029 z m -4.472595,7.10551 v 1.789038 h 1.7890381 v -1.789038 z m 3.578076,0 v 1.789038 h 1.789038 v -1.789038 z m 3.57756,0 v 1.789038 h 1.789038 v -1.789038 z" + }, + { + "keys": [ "Advanced", "Rocket" ], + "path": "M 21.937636,2 21.867156,2.073804 20.80291,2.1878302 C 18.062974,2.4818921 13.899693,3.2082055 10.390717,6.6042293 9.1339847,6.5797643 7.7103209,6.7338687 6.4086517,7.2669845 5.0027015,7.8428111 3.7202379,8.9408345 3.2550298,10.639434 L 3.0232869,11.48928 6.49943,12.379719 6.6315382,12.813684 5.9263461,13.359091 c -0.3114304,0.241012 -0.6014418,0.645379 -0.4963303,0.964611 0.4820822,1.464129 3.3722308,4.237011 5.0023982,4.782095 0.318745,0.106579 0.714621,-0.217169 0.941364,-0.539134 l 0.512934,-0.729178 0.439499,0.112181 1.046164,3.432599 0.838408,-0.269752 c 1.675927,-0.541228 2.715126,-1.871881 3.227056,-3.302336 0.473959,-1.324355 0.564077,-2.75317 0.483044,-4.007527 3.234561,-3.658347 3.772203,-7.8503652 3.942581,-10.6007606 l 0.06605,-1.0683052 0.07048,-0.073803 -0.06642,0.00738 z M 19.601757,4.5067346 C 19.270801,7.9284136 18.17382,12.187959 12.393372,15.427064 L 9.0164951,12.1989 C 11.9923,6.2778655 16.198274,4.99133 19.601757,4.5067346 Z m -4.9083,2.4709398 c -0.654976,-0.00753 -1.302749,0.2325479 -1.784198,0.7361895 -0.962899,1.0072825 -0.842475,2.6310231 0.177866,3.6064061 1.020342,0.975383 2.648673,1.023518 3.611572,0.01624 0.9629,-1.007283 0.842475,-2.6328684 -0.177866,-3.6082514 C 16.01066,7.2405642 15.348434,6.9852037 14.693457,6.9776744 Z m 0.0015,1.4203476 c 0.286756,0.00565 0.587913,0.122211 0.839146,0.362375 0.502465,0.4803252 0.520776,1.1831605 0.132477,1.589358 -0.388299,0.406196 -1.090212,0.418326 -1.592679,-0.06199 C 13.571432,9.8074335 13.553123,9.1071813 13.94142,8.7009852 14.13557,8.4978867 14.408198,8.3923746 14.694953,8.398022 Z M 8.7582069,8.459279 C 8.2406887,9.1470208 7.7482786,9.9113689 7.289498,10.762316 L 5.4587992,10.294033 C 5.8515306,9.6762833 6.3709104,9.1899043 7.1123657,8.8862319 7.6142475,8.6806798 8.1806174,8.5423885 8.7581829,8.4592789 Z m -1.1343832,5.384331 3.1894163,3.048821 -0.538027,0.763866 c -0.04033,9.28e-4 -0.142939,-0.0057 -0.2952138,-0.0631 C 9.0506641,16.997259 7.4543968,15.816163 6.9614377,14.70748 6.8972622,14.557943 6.8859608,14.455911 6.8850511,14.415587 Z M 5.5783608,15.510462 5.26322,15.745157 c -1.9976954,1.48272 -2.562579,3.791931 -3.1082328,5.833056 L 2,22.157939 2.5730839,22.045758 C 5.8165767,21.409084 7.3740648,20.68256 8.7113177,19.275914 L 9.0430643,18.926824 8.629396,18.686225 C 7.3615243,17.951377 6.4446985,16.937858 5.7846415,15.849589 Z m 10.5627512,0.0063 c -0.05702,0.580725 -0.169669,1.152904 -0.352411,1.663533 -0.269977,0.754377 -0.732716,1.295039 -1.332153,1.715192 l -0.550205,-1.807816 c 0.829425,-0.496633 1.571028,-1.022963 2.234769,-1.570906 z M 5.4399793,16.601646 c 0.6009312,0.899031 1.3946322,1.723037 2.3993505,2.400826 -1.0473448,0.990391 -2.3817924,1.601785 -4.7743458,2.127385 0.4662491,-1.708979 1.0173783,-3.36773 2.3749953,-4.528211 z M 5.0798184,17.795048 3.8742389,20.457508 6.456991,19.20248 6.1470165,18.524226 5.3864718,18.893612 5.7392523,18.112772 Z" + }, + { + "keys": [ "Staged", "Inbox" ], + "path": "M 19,20 H 5 C 3.89,20 3,19.1 3,18 V 6 C 3,4.9 3.89,4 5,4 H 9 V 6 H 5 V 16 H 19 V 6 H 15 V 4 h 4 c 1.1,0 2,0.9 2,2 v 12 c 0,1.1 -0.89,2 -2,2 z M 12,14 8,10 h 3 V 4 h 2 v 6 h 3 z" + }, + { + "keys": [ "Alias" ], + "path": "m 18.460839,3.00001 c -0.9606,0.004 -1.8258,1.29582 -0.9632,2.23063 l 0.3016,0.3267 c -1.267,0.0907 -2.4872,0.41304 -3.6138,1.14529 -0.8417,0.54721 -1.5495,1.25043 -2.1809,2.02092 0.1268,0.17784 0.2535,0.35855 0.3781,0.54721 0.1993,0.30163 0.3948,0.60913 0.5841,0.90628 0.1789,0.28134 0.3557,0.55798 0.5339,0.82915 0.3268,-0.45118 0.6754,-0.88674 1.0677,-1.29173 0.9902,-1.02211 2.1183,-1.44209 3.357,-1.56092 -0.079,0.0947 -0.1549,0.19109 -0.2341,0.28529 -0.4526,0.53777 -0.5289,1.29624 0,1.82533 0.4653,0.46543 1.3719,0.5389 1.8253,0 0.6983,-0.82958 1.3653,-1.68547 2.0636,-2.51505 0.4547,-0.45705 0.6288,-1.21716 0.066,-1.82688 l -2.3231,-2.51689 c -0.2651,-0.28721 -0.5681,-0.4066 -0.8622,-0.40533 z M 3.7938394,4.80711 c -0.1798,0 -0.3627,0.005 -0.5483,0.014 -1.6294,0.0828 -1.6636,2.58328 -0.079,2.58328 0.026,0 0.052,-7.8e-4 0.079,-0.002 0.1789,-0.009 0.3561,-0.0138 0.5322,-0.0138 1.6492,0 3.1669,0.42116 4.3566,1.67286 0.7315,0.7697 1.3087,1.65133 1.8738996,2.54239 0.2681,0.42255 0.5337,0.84679 0.8119,1.26291 0.2392,0.35785 0.4872,0.71333 0.7465,1.05997 0.7391,0.98756 1.5775,1.90122 2.6187,2.57812 1.1266,0.7323 2.3467,1.0546 3.6138,1.14507 l -0.3016,0.32678 c -0.8642,0.93642 0.01,2.23066 0.9677,2.23066 0.2926,0 0.594,-0.11984 0.8576,-0.40531 l 2.3233,-2.51682 c 0.5629,-0.60981 0.3888,-1.36992 -0.066,-1.82688 -0.6981,-0.83002 -1.365,-1.68541 -2.0634,-2.51521 -0.2186,-0.25948 -0.5421,-0.37706 -0.8711,-0.37706 -0.3534,0 -0.7133,0.13613 -0.9542,0.37706 -0.5289,0.5293 -0.4525,1.28765 0,1.82533 0.079,0.0942 0.1556,0.19086 0.2341,0.2855 -1.2387,-0.11866 -2.367,-0.539 -3.357,-1.56102 -0.5633,-0.5814 -1.0385,-1.22432 -1.4847,-1.8891 -0.4324,-0.64414 -0.8371,-1.30904 -1.2664,-1.95935 -0.082,-0.12423 -0.1664,-0.2435 -0.2502,-0.36372 C 9.6415394,6.51829 7.2332394,4.80717 3.7938394,4.80717 Z m 5.8102,7.42417 c -0.4424,0.67295 -0.9108,1.3267 -1.4697,1.9147 -1.3167,1.38513 -3.0351,1.75293 -4.889,1.65873 -1.6645,-0.0847 -1.6564,2.49717 0,2.58142 3.4832,0.17693 5.9452,-1.32799 7.9053996,-3.88999 -0.3228,-0.42077 -0.6211,-0.84458 -0.8941,-1.253 -0.2246,-0.33561 -0.4418996,-0.67869 -0.6525996,-1.01186 z" + }, + { + "keys": [ "Trakt" ], + "path": "m 8.5148718,3.0007766 c -1.0450158,0.0016 -2.0347477,0.00569 -2.8165557,0.014198 -0.9749586,0.010626 -1.7535238,0.525551 -2.41504,1.1480853 -0.6615168,0.6225342 -1.2420491,1.3803092 -1.255398,2.3912657 -0.054734,4.1452564 -0.014494,8.0973684 0,11.3576864 0.00425,0.95513 0.5049474,1.64923 1.0794047,2.298813 0.5744572,0.64958 1.2158261,1.27916 2.248952,1.294359 3.8273691,0.05632 7.1951532,0.05445 11.0869282,0 0.900735,-0.01259 1.655548,-0.434309 2.373765,-0.966147 0.718218,-0.531839 1.46119,-1.152567 1.48026,-2.257537 0.01521,-0.881315 0,-2.321265 0,-2.321265 l -1.917106,0.02014 c 0,0 0.01418,1.445506 0,2.267444 0.0021,-0.123048 -0.217365,0.390857 -0.703974,0.751191 -0.48661,0.360334 -1.179113,0.58793 -1.260351,0.589066 -3.876942,0.05424 -7.2205623,0.05606 -11.0307952,0 0.088118,0.0013 -0.460611,-0.21638 -0.8410046,-0.64652 -0.3803932,-0.430138 -0.5993469,-1.048157 -0.5993021,-1.03813 -0.014538,-3.269123 -0.054357,-7.207571 0,-11.3243374 0.00121,-0.090137 0.2320137,-0.6233247 0.6527937,-1.0193089 0.4207806,-0.3959843 0.9958328,-0.6266544 1.1219997,-0.6280294 3.0843491,-0.033603 9.5518555,0 9.5518555,0 l 0.01023,-1.916777 c 0,0 -3.631619,-0.01899 -6.7666673,-0.014198 z M 20.503229,3.1183256 9.6966363,13.838738 5.6732212,9.8215966 4.7708006,10.725008 12.311113,18.256406 13.214855,17.351674 10.601369,14.742149 21.276215,4.1508427 C 21.059818,3.7752637 20.800627,3.4289481 20.503229,3.1183256 Z m 1.062566,1.6050741 -10.014458,9.9778053 2.182583,2.218905 0.911006,-0.894827 -1.29172,-1.315493 8.594953,-8.5635841 C 21.883198,5.6470001 21.755599,5.1687836 21.565795,4.7233997 Z M 22,7.0080126 14.287986,14.653657 15.187765,15.561031 22,8.8075706 Z M 8.4214271,7.119618 7.516365,8.0220384 l 3.48685,3.4954346 0.905062,-0.90242 z M 7.0359324,8.4862915 6.1325213,9.3897019 9.5969182,12.85641 10.500329,11.952999 Z" } ] \ No newline at end of file diff --git a/frontend/scss/app/_component.scss b/frontend/scss/app/_component.scss index da9d850e9..09448ae6b 100644 --- a/frontend/scss/app/_component.scss +++ b/frontend/scss/app/_component.scss @@ -80,7 +80,6 @@ @forward "components/source_editor/source_editor"; @forward "components/source_editor/source_editor_form"; @forward "components/source_editor/source_editor_block"; -@forward "components/source_editor/input_form"; @forward "components/cluster_flags"; @forward "components/titled_card"; @forward "components/filter_input"; diff --git a/frontend/scss/app/components/_form.scss b/frontend/scss/app/components/_form.scss index 646e62dcc..a58f7ff52 100644 --- a/frontend/scss/app/components/_form.scss +++ b/frontend/scss/app/components/_form.scss @@ -35,6 +35,7 @@ } label { + padding-left: 2px; font-weight: bold; white-space: nowrap; vertical-align: middle; diff --git a/frontend/scss/app/components/_input.scss b/frontend/scss/app/components/_input.scss index 7cc02f8e2..c50353133 100644 --- a/frontend/scss/app/components/_input.scss +++ b/frontend/scss/app/components/_input.scss @@ -3,7 +3,7 @@ flex-flow: column; overflow: hidden; box-sizing: border-box; - gap: var(--gap-small); + gap: var(--gap-default); .tp__input-wrapper { display: flex; flex-flow: row nowrap; diff --git a/frontend/scss/app/components/_menu_item.scss b/frontend/scss/app/components/_menu_item.scss index 44b93ba0a..1d0fd01c0 100644 --- a/frontend/scss/app/components/_menu_item.scss +++ b/frontend/scss/app/components/_menu_item.scss @@ -21,6 +21,6 @@ &:hover { background-color: var(--menu-item-hover-background-color); - color: var(--menu-item-hover-color) + color: var(--menu-item-hover-color); } } \ No newline at end of file diff --git a/frontend/scss/app/components/_radio_button_group.scss b/frontend/scss/app/components/_radio_button_group.scss index 116897605..5d83c7538 100644 --- a/frontend/scss/app/components/_radio_button_group.scss +++ b/frontend/scss/app/components/_radio_button_group.scss @@ -1,6 +1,16 @@ .tp__radio-button-group { display: flex; flex-flow: row nowrap; - gap: var(--gap-small); + + button:not(:first-child) { + border-top-left-radius: 0 !important; + border-bottom-left-radius: 0 !important; + border-right: 0 !important; + } + button:not(:last-child) { + border-top-right-radius: 0 !important; + border-bottom-right-radius: 0 !important; + border-left: 0 !important; + } } \ No newline at end of file diff --git a/frontend/scss/app/components/_sidebar.scss b/frontend/scss/app/components/_sidebar.scss index 36d60986f..64c8cdbf5 100644 --- a/frontend/scss/app/components/_sidebar.scss +++ b/frontend/scss/app/components/_sidebar.scss @@ -61,7 +61,7 @@ .tp__menu-item.active, .tp__icon-button.active { background-color: var(--menu-item-hover-background-color); - color: var(--menu-item-hover-color) + color: var(--menu-item-hover-color); } &__content { diff --git a/frontend/scss/app/components/api_user/_api_user_view.scss b/frontend/scss/app/components/api_user/_api_user_view.scss index fde91d0ac..e31df2035 100644 --- a/frontend/scss/app/components/api_user/_api_user_view.scss +++ b/frontend/scss/app/components/api_user/_api_user_view.scss @@ -72,6 +72,10 @@ .tp__collapse-panel__header { font-size: 1.6rem; + padding: 0; + span { + pointer-events: auto; + } } > :nth-child(1) { @@ -133,4 +137,28 @@ } } + &__section-header { + display: flex; + flex-flow: row nowrap; + justify-content: space-between; + gap: var(--gap-default); + box-sizing: border-box; + overflow: hidden; + width: 100%; + + &__toolbar { + display: flex; + flex-flow: row nowrap; + gap: var(--gap-default); + box-sizing: border-box; + .tp__icon-button:hover, + .tp__icon-button:focus-visible { + background-color: var(--action-item-background-color); + } + svg, img { + height: 2rem; + width: 2rem; + } + } + } } \ No newline at end of file diff --git a/frontend/scss/app/components/config/_config_view.scss b/frontend/scss/app/components/config/_config_view.scss index be6a5c76a..666f5f1ad 100644 --- a/frontend/scss/app/components/config/_config_view.scss +++ b/frontend/scss/app/components/config/_config_view.scss @@ -1,5 +1,14 @@ @use "../../../size" as size; +%toolbar-base { + display: flex; + flex-flow: row; + gap: var(--gap-large); + box-sizing: border-box; + width: 100%; + border-top: 1px solid var(--card-border-color); +} + .tp__config-view { display: flex; flex-flow: column; @@ -43,13 +52,7 @@ } &__toolbar { - padding-top: var(--padding-default); - display: flex; - flex-flow: row; - gap: var(--gap-large); - box-sizing: border-box; - width: 100%; - border-top: 1px solid var(--card-border-color); + @extend %toolbar-base; } &__tags { @@ -131,7 +134,6 @@ &__body { display: grid; grid-template-columns: repeat(auto-fit, minmax(size.$config-grid-cell-width, 1fr)); - //max-width: calc(size.$config-max-grid-cells * size.$config-grid-cell-width + 2 * 1rem); gap: var(--gap-large); box-sizing: border-box; width: 100%; @@ -292,4 +294,8 @@ gap: var(--gap-default); box-sizing: border-box; } -} \ No newline at end of file +} + +.tp__form-page__toolbar { + @extend %toolbar-base; +} diff --git a/frontend/scss/app/components/playlist/_playlist_explorer.scss b/frontend/scss/app/components/playlist/_playlist_explorer.scss index 3bf444925..3e8605587 100644 --- a/frontend/scss/app/components/playlist/_playlist_explorer.scss +++ b/frontend/scss/app/components/playlist/_playlist_explorer.scss @@ -1,6 +1,39 @@ $movie-card-height: 24rem; $movie-card-width: 16rem; $live-card-width: 17.5rem; +$episode-card-height: 10rem; +$episode-card-width: 18rem; +$series-backdrop-height: 300px; + +@mixin card-base($height, $width) { + position: relative; + padding: 0; + max-height: $height; + min-height: $height; + max-width: $width; + min-width: $width; + + + .tp__playlist-explorer__channel { + &-logo { + box-sizing: border-box; + height: 100%; + object-fit: cover; + object-position: center; + } + + &-title { + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 3rem; + background-color: var(--card-background-color); + display: flex; + align-items: center; + } + } +} .tp__playlist-explorer { display: flex; @@ -29,7 +62,7 @@ $live-card-width: 17.5rem; width: 100%; justify-content: space-between; - &-actions { + &-actions { display: flex; flex-flow: row; gap: var(--gap-default); @@ -71,7 +104,7 @@ $live-card-width: 17.5rem; box-sizing: border-box; overflow: hidden; - &-sidebar{ + &-sidebar { .tp__icon-button.active { color: var(--tag-active-color); fill: currentColor; @@ -101,6 +134,7 @@ $live-card-width: 17.5rem; box-sizing: border-box; width: 100%; } + &-video, &-series { display: grid; @@ -109,6 +143,14 @@ $live-card-width: 17.5rem; box-sizing: border-box; width: 100%; } + + &-episodes { + display: grid; + grid-template-columns: repeat(auto-fill, minmax($episode-card-width, 1fr)); + gap: var(--gap-default); + box-sizing: border-box; + width: 100%; + } } } @@ -203,40 +245,16 @@ $live-card-width: 17.5rem; &-logo { box-sizing: border-box; - max-height: 3rem; - min-height: 3rem; - aspect-ratio: 1 / 1; + height: 3rem; + object-fit: cover; + object-position: center; } } } &-video, &-series { - position: relative; - max-height: $movie-card-height; - min-height: $movie-card-height; - max-width: $movie-card-width; - min-width: $movie-card-width; - padding:0; - .tp__playlist-explorer__channel { - &-logo { - box-sizing: border-box; - width: 100%; - height: 100%; - aspect-ratio: 1 / 1; - } - &-title { - position: absolute; - bottom: 0; - left: 0; - right: 0; - height: 3rem; - background-color: var(--card-background-color); - padding-left: 2rem; - display: flex; - align-items: center; - } - } + @include card-base($movie-card-height, $movie-card-width); .tp__icon-button { position: absolute; z-index: 1; @@ -246,6 +264,78 @@ $live-card-width: 17.5rem; } } + &-episode { + @include card-base($episode-card-height, $episode-card-width); + } } + &__series-info { + display: flex; + flex-flow: column; + gap: var(--gap-default); + box-sizing: border-box; + overflow: hidden; + + &__nowrap { + white-space: nowrap; + } + + &__body { + display: flex; + flex-flow: column; + gap: var(--gap-default); + box-sizing: border-box; + overflow: auto; + + .tp__playlist-explorer__group-list { + overflow: unset; + } + + &-top { + display: flex; + flex-flow: column; + gap: var(--gap-default); + min-height: $series-backdrop-height; + background-size: cover; + background-position: right top; + + font-size: 1.2rem; + box-sizing: border-box; + + &-content { + display: flex; + flex-flow: column; + gap: var(--gap-default); + max-width: 60%; + background-size: cover; + background-position: center; + font-size: 1.2rem; + } + } + } + + &__title { + font-weight: bold; + font-size: 2.5rem; + color: var(--series-info-title-color, white); + } + + &__infos { + display: flex; + flex-flow: row; + gap: var(--gap-larger); + } + + &__season { + color: var(--series-info-title-color, white); + display: flex; + flex-flow: column; + gap: var(--gap-default); + + &-title { + font-weight: bold; + font-size: 2rem; + } + } + } } \ No newline at end of file diff --git a/frontend/scss/app/components/source_editor/_input_form.scss b/frontend/scss/app/components/source_editor/_input_form.scss deleted file mode 100644 index e69de29bb..000000000 diff --git a/frontend/scss/app/components/source_editor/_source_editor.scss b/frontend/scss/app/components/source_editor/_source_editor.scss index 7f41e3324..ecde03532 100644 --- a/frontend/scss/app/components/source_editor/_source_editor.scss +++ b/frontend/scss/app/components/source_editor/_source_editor.scss @@ -101,7 +101,7 @@ } &__selection-rect { - position:absolute; + position: absolute; display: none; border: 1px dashed var(--selection-area-border-color); background: var(--selection-area-background-color); @@ -109,8 +109,12 @@ pointer-events: none; } + .tp__form-page__toolbar { + padding: var(--padding-default) 0 var(--padding-default) var(--padding-small); + } } + .graph-paper-advanced { background-image: linear-gradient(var(--graph-paper-line-color-1) 1px, transparent 1px), @@ -132,14 +136,3 @@ .clickable { cursor: pointer; } - -.tp__form-page__toolbar { - margin-top: var(--padding-default); - padding-top: var(--padding-default); - display: flex; - flex-flow: row; - gap: var(--gap-large); - box-sizing: border-box; - width: 100%; - border-top: 1px solid var(--card-border-color); -} \ No newline at end of file diff --git a/frontend/scss/app/components/source_editor/_source_editor_form.scss b/frontend/scss/app/components/source_editor/_source_editor_form.scss index 65cdbcfc1..30c49a544 100644 --- a/frontend/scss/app/components/source_editor/_source_editor_form.scss +++ b/frontend/scss/app/components/source_editor/_source_editor_form.scss @@ -1,4 +1,4 @@ -$form-width: 380px; +$form-width: 480px; .tp__source-editor-form-wrapper { position: absolute; @@ -21,12 +21,17 @@ $form-width: 380px; .tp__source-editor-form { + padding: 0 var(--padding-micro); + background-color: var(--card-background-color); + &__body { background-color: var(--card-background-color); border: 1px solid var(--card-border-color); display: flex; flex-flow: column; + width: 100%; + overflow: hidden; &__pages { display: flex; @@ -39,15 +44,40 @@ $form-width: 380px; } } + &__content { + display: flex; + flex-flow: row; + gap: var(--gap-default); + box-sizing: border-box; + overflow: hidden; + } + .tp__form-list__item { display: flex; flex-flow: row nowrap; gap: var(--gap-small); align-items: center; + + &-toolbar { + display: flex; + flex-flow: row nowrap; + } + } + + .tp__form-list__toolbar { + margin-top: var(--margin-default); + } + + &__sidebar { + .tp__icon-button.active { + background-color: var(--menu-item-hover-background-color); + color: var(--menu-item-hover-color); + } + .tp__icon-button { - fill: var(--menu-button-warn-color); - &:hover { - fill: var(--menu-button-warn-hover-color); + .svg-icon { + width: 2rem; + height: 2rem; } } } diff --git a/frontend/src/app/components/api_user/target_playlist.rs b/frontend/src/app/components/api_user/target_playlist.rs index 76e1914b9..07567357d 100644 --- a/frontend/src/app/components/api_user/target_playlist.rs +++ b/frontend/src/app/components/api_user/target_playlist.rs @@ -1,12 +1,15 @@ use std::cell::RefCell; -use crate::app::components::{AppIcon, Card, CollapsePanel}; +use crate::app::components::{AppIcon, Card, CollapsePanel, IconButton, RadioButtonGroup}; use shared::model::{PlaylistClusterBouquetDto, PlaylistClusterCategoriesDto, XtreamCluster}; use std::collections::HashMap; +use std::fmt::{Display, Formatter}; use std::rc::Rc; use std::str::FromStr; use wasm_bindgen::JsCast; use yew::prelude::*; use yew_i18n::use_translation; +use shared::error::TuliproxError; +use shared::info_err_res; use crate::html_if; fn normalize(s: &str) -> String { @@ -39,6 +42,44 @@ macro_rules! create_selection { }; } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum FilterState { + All, + Selected, + Deselected, +} + +impl FilterState { + const ALL: &'static str = "All"; + const SELECTED: &'static str = "Selected"; + const DESELECTED: &'static str = "Deselected"; +} + +impl Display for FilterState { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", + match self { + Self::All => Self::ALL, + Self::Selected => Self::SELECTED, + Self::Deselected => Self::DESELECTED, + } + ) + } +} + +impl FromStr for FilterState { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s { + Self::ALL => Ok(Self::All), + Self::SELECTED => Ok(Self::Selected), + Self::DESELECTED => Ok(Self::Deselected), + _ => info_err_res!("Unknown FilterState: {s}"), + } + } +} + #[derive(Clone, PartialEq, Default)] pub struct BouquetSelection { pub live: HashMap, @@ -58,6 +99,8 @@ pub fn UserTargetPlaylist(props: &UserTargetPlaylistProps) -> Html { let translate = use_translation(); let bouquet_selection = use_mut_ref(BouquetSelection::default); let playlist_categories = use_state(PlaylistClusterCategoriesDto::default); + let filter_state = use_state(HashMap::::new); + let collapse_state = use_state(HashMap::::new); let force_update = use_state(|| 0); { @@ -107,6 +150,7 @@ pub fn UserTargetPlaylist(props: &UserTargetPlaylistProps) -> Html { let force_update = force_update.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); if let Some(target) = e.target() { if let Ok(element) = target.dyn_into::() { if let Some(cluster) = element.get_attribute("data-cluster") { @@ -137,17 +181,112 @@ pub fn UserTargetPlaylist(props: &UserTargetPlaylistProps) -> Html { }) }; + let handle_selection_change = { + let bouquet_selection = bouquet_selection.clone(); + let on_change = props.on_change.clone(); + let force_update = force_update.clone(); + + Callback::from(move |(cluster, cats, select): (XtreamCluster, Vec, bool)| { + { + let mut selections = bouquet_selection.borrow_mut(); + let map = match cluster { + XtreamCluster::Live => &mut selections.live, + XtreamCluster::Video => &mut selections.vod, + XtreamCluster::Series => &mut selections.series, + }; + for cat in cats { + map.insert(cat, select); + } + } + on_change.emit(bouquet_selection.clone()); + force_update.set(*force_update + 1); + }) + }; + let render_category_cluster = |cluster: XtreamCluster, cats: Option<&Vec>, selections: &HashMap| { if let Some(c) = cats { + let cats_clone = c.clone(); + let cluster_clone = cluster; + let handler = handle_selection_change.clone(); + let current_filter = *filter_state.get(&cluster).unwrap_or(&FilterState::All); + + let select_all = { + let cats = cats_clone.clone(); + let handler = handler.clone(); + Callback::from(move |(_, event): (String, MouseEvent)| { + event.stop_propagation(); + handler.emit((cluster_clone, cats.clone(), true)); + }) + }; + + let deselect_all = { + let cats = cats_clone.clone(); + let handler = handler.clone(); + Callback::from(move |(_, event): (String, MouseEvent)| { + event.stop_propagation(); + handler.emit((cluster_clone, cats.clone(), false)); + }) + }; + + let filter_state_handle = filter_state.clone(); + let filter_state_selections = Rc::new(vec![filter_state.get(&cluster).cloned().unwrap_or(FilterState::All).to_string()]); + let title_content = if *collapse_state.get(&cluster).unwrap_or(&true) { + html! { +

+
+ {translate.t( match cluster { + XtreamCluster::Live => "LABEL.LIVE", + XtreamCluster::Video => "LABEL.MOVIE", + XtreamCluster::Series => "LABEL.SERIES" + })} +
+
+ >| { + if let Some(first) = selections.first() { + let mut cluster_state = (*filter_state_handle).clone(); + cluster_state.insert(cluster, FilterState::from_str(first.as_str()).unwrap_or(FilterState::All)); + filter_state_handle.set(cluster_state); + } + })} + options={Rc::new([FilterState::All, FilterState::Selected, FilterState::Deselected].iter().map(|s| s.to_string()).collect::>())} + selected={filter_state_selections} + /> + + +
+
+ } + } else { + html! { +
+ {translate.t( match cluster { + XtreamCluster::Live => "LABEL.LIVE", + XtreamCluster::Video => "LABEL.MOVIE", + XtreamCluster::Series => "LABEL.SERIES" + })} +
+ } + }; + + let collapse_state = collapse_state.clone(); html_if!(!c.is_empty(), { - "LABEL.LIVE", - XtreamCluster::Video => "LABEL.MOVIE", - XtreamCluster::Series => "LABEL.SERIES" +
- { for c.iter().map(|cat| { + { for c.iter().filter(|cat| { + let selected = *selections.get(*cat).unwrap_or(&false); + match current_filter { + FilterState::All => true, + FilterState::Selected => selected, + FilterState::Deselected => !selected, + } + }).map(|cat| { let selected = *selections.get(cat).unwrap_or(&false); html! {
Html { let click = props.onclick.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); if let Some(target) = e.target() { if let Ok(element) = target.dyn_into::() { if let Some(data_name) = element.get_attribute("data-name") { diff --git a/frontend/src/app/components/collapse_panel.rs b/frontend/src/app/components/collapse_panel.rs index 0358378f2..ee5abc006 100644 --- a/frontend/src/app/components/collapse_panel.rs +++ b/frontend/src/app/components/collapse_panel.rs @@ -12,6 +12,8 @@ pub struct CollapsePanelProps { pub children: Children, #[prop_or_default] pub class: String, + #[prop_or_default] + pub on_state_change: Callback, } #[function_component] @@ -21,7 +23,12 @@ pub fn CollapsePanel(props: &CollapsePanelProps) -> Html { let toggle = { let expanded = expanded.clone(); - Callback::from(move |_| expanded.set(!*expanded)) + let on_state_change = props.on_state_change.clone(); + Callback::from(move |_| { + let new_state = !*expanded; + expanded.set(new_state); + on_state_change.emit(new_state); + }) }; html! { diff --git a/frontend/src/app/components/config/macros.rs b/frontend/src/app/components/config/macros.rs index 5134803c5..a624aa748 100644 --- a/frontend/src/app/components/config/macros.rs +++ b/frontend/src/app/components/config/macros.rs @@ -185,9 +185,9 @@ macro_rules! edit_field_text { hidden={$hidden} name={stringify!($field)} autocomplete={true} - value={instance.form.$field.clone()} + value={instance.form.$field.to_string()} on_change={Callback::from(move |value: String| { - instance.dispatch($action(value)); + instance.dispatch($action(value.into())); })} />
@@ -498,7 +498,7 @@ macro_rules! generate_form_reducer { $( $action_name::$set_name(v) => { let mut new_data = self.$data_field.clone(); - new_data.$field_name = v; + new_data.$field_name = v.into(); if !modified { modified = true; } new_data }, diff --git a/frontend/src/app/components/dashboard/stats_view.rs b/frontend/src/app/components/dashboard/stats_view.rs index 3830451a0..1713120d6 100644 --- a/frontend/src/app/components/dashboard/stats_view.rs +++ b/frontend/src/app/components/dashboard/stats_view.rs @@ -25,7 +25,7 @@ pub fn StatsView() -> Html { html! { diff --git a/frontend/src/app/components/dashboard/streams_table.rs b/frontend/src/app/components/dashboard/streams_table.rs index 65c691687..671abbf87 100644 --- a/frontend/src/app/components/dashboard/streams_table.rs +++ b/frontend/src/app/components/dashboard/streams_table.rs @@ -180,7 +180,7 @@ pub fn StreamsTable(props: &StreamsTableProps) -> Html { } } - "USERNAME" => html! {dto.username.as_str()}, + "USERNAME" => html! {&dto.username}, "STREAM_ID" => html! { <> { dto.channel.virtual_id.to_string() } {" ("} @@ -188,13 +188,13 @@ pub fn StreamsTable(props: &StreamsTableProps) -> Html { {")"} }, "CLUSTER" => html! { render_cluster(&dto.channel) }, - "CHANNEL" => html! {dto.channel.title.as_str()}, + "CHANNEL" => html! {&dto.channel.title}, "GROUP" => html! {&*dto.channel.group}, "CLIENT_IP" => html! { strip_port(&dto.client_ip)}, "COUNTRY" => html! { dto.country.as_ref().map_or_else(String::new, |c| t_safe(&translate, &format!("COUNTRY.{c}")).unwrap_or_else(||c.to_string())) }, - "PROVIDER" => html! {dto.provider.as_str()}, + "PROVIDER" => html! {&dto.provider}, "SHARED" => html! { }, - "USER_AGENT" => html! { {dto.user_agent.as_str()} }, + "USER_AGENT" => html! { {&dto.user_agent} }, "DURATION" => html! { {format_duration(current_time_secs() - dto.ts)} }, _ => html! {""}, } @@ -268,7 +268,7 @@ pub fn StreamsTable(props: &StreamsTableProps) -> Html { } StreamsTableAction::CopyLinkProviderUrl => { if let Some(dto) = &*selected_dto { - copy_to_clipboard.emit(dto.channel.url.clone()); + copy_to_clipboard.emit(dto.channel.url.to_string()); } } StreamsTableAction::CopyLinkTuliproxWebPlayerUrl => { diff --git a/frontend/src/app/components/drop_down_icon_button.rs b/frontend/src/app/components/drop_down_icon_button.rs index 2b167f229..9e88707c4 100644 --- a/frontend/src/app/components/drop_down_icon_button.rs +++ b/frontend/src/app/components/drop_down_icon_button.rs @@ -71,6 +71,7 @@ pub fn DropDownIconButton(props: &DropDownIconButtonProps) -> Html { let set_is_open = popup_is_open.clone(); Callback::from(move |(_name, event): (String, MouseEvent)| { event.prevent_default(); + event.stop_propagation(); if let Some(button) = button_ref.cast::() { set_anchor_ref.set(Some(button)); set_is_open.set(true); @@ -86,6 +87,7 @@ pub fn DropDownIconButton(props: &DropDownIconButtonProps) -> Html { let set_is_open = popup_is_open.clone(); Callback::from(move |(id, e): (String, MouseEvent)| { e.prevent_default(); + e.stop_propagation(); if selections.current().contains(&id) { selections.remove(&id); } else { diff --git a/frontend/src/app/components/filter/filter_input.rs b/frontend/src/app/components/filter/filter_input.rs index 4ccb2e9cb..f0575b1a1 100644 --- a/frontend/src/app/components/filter/filter_input.rs +++ b/frontend/src/app/components/filter/filter_input.rs @@ -78,6 +78,7 @@ pub fn FilterInput(props: &FilterInputProps) -> Html { let dialog_actions = dialog_actions.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); let original_filter = (*filter_state).clone(); let original_templates = (*templates_state).clone(); let current_filter = (*filter_state).clone(); diff --git a/frontend/src/app/components/hide_content.rs b/frontend/src/app/components/hide_content.rs index cc11d42db..5ba3cd55e 100644 --- a/frontend/src/app/components/hide_content.rs +++ b/frontend/src/app/components/hide_content.rs @@ -17,6 +17,7 @@ pub fn HideContent(props: &HideContentProps) -> Html { let hidden = hidden.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); hidden.set(!*hidden); }) }; diff --git a/frontend/src/app/components/home.rs b/frontend/src/app/components/home.rs index 66cb14700..79044ae26 100644 --- a/frontend/src/app/components/home.rs +++ b/frontend/src/app/components/home.rs @@ -6,6 +6,7 @@ use shared::model::{AppConfigDto, ConfigInputDto, LibraryScanSummaryStatus, Play use std::collections::HashMap; use std::future; use std::rc::Rc; +use std::sync::Arc; use yew::prelude::*; use yew::suspense::use_future; use yew_i18n::use_translation; @@ -105,7 +106,7 @@ pub fn Home() -> Html { if let Some(cfg) = config_ctx.as_ref() { let mut sources = vec![]; // Create a map for a faster lookup of global inputs by name - let inputs_map: HashMap = cfg.sources.inputs.iter().map(|i| (i.name.clone(), i)).collect(); + let inputs_map: HashMap, &ConfigInputDto> = cfg.sources.inputs.iter().map(|i| (i.name.clone(), i)).collect(); for source in &cfg.sources.sources { let mut inputs = vec![]; diff --git a/frontend/src/app/components/icon_button.rs b/frontend/src/app/components/icon_button.rs index a37188816..882618946 100644 --- a/frontend/src/app/components/icon_button.rs +++ b/frontend/src/app/components/icon_button.rs @@ -10,6 +10,8 @@ pub struct IconButtonProps { #[prop_or_default] pub class: String, #[prop_or_default] + pub hint: String, + #[prop_or_default] pub button_ref: Option, } @@ -21,12 +23,13 @@ pub fn IconButton(props: &IconButtonProps) -> Html { let name = props.name.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); click.emit((name.clone(), e)); }) }; html! { - } diff --git a/frontend/src/app/components/key_value_editor.rs b/frontend/src/app/components/key_value_editor.rs index c1cef7ef8..ca28284af 100644 --- a/frontend/src/app/components/key_value_editor.rs +++ b/frontend/src/app/components/key_value_editor.rs @@ -96,6 +96,7 @@ pub fn KeyValueEditor(props: &KeyValueEditorProps) -> Html { Callback::from(move |e: KeyboardEvent| { if e.key() == "Enter" { e.prevent_default(); + e.stop_propagation(); let key = (*new_key).trim().to_string(); let value = (*new_value).trim().to_string(); if !key.is_empty() && !value.is_empty() && !entry_state.iter().any(|kv| kv.key == key) { diff --git a/frontend/src/app/components/login.rs b/frontend/src/app/components/login.rs index 018fc98be..66fbb3bd1 100644 --- a/frontend/src/app/components/login.rs +++ b/frontend/src/app/components/login.rs @@ -61,6 +61,7 @@ pub fn Login() -> Html { Callback::from(move |e: KeyboardEvent| { if e.key() == "Enter" { e.prevent_default(); + e.stop_propagation(); authenticator.run(); } }) diff --git a/frontend/src/app/components/menu_item.rs b/frontend/src/app/components/menu_item.rs index c98680368..5397a2cac 100644 --- a/frontend/src/app/components/menu_item.rs +++ b/frontend/src/app/components/menu_item.rs @@ -23,6 +23,7 @@ pub fn MenuItem(props: &MenuItemProps) -> Html { let name = props.name.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); click.emit((name.clone(), e)); }) }; diff --git a/frontend/src/app/components/number_input.rs b/frontend/src/app/components/number_input.rs index fa0bde0f7..37ced43a8 100644 --- a/frontend/src/app/components/number_input.rs +++ b/frontend/src/app/components/number_input.rs @@ -87,6 +87,7 @@ pub fn NumberInput(props: &NumberInputProps) -> Html { if !allowed { e.prevent_default(); + e.stop_propagation(); } }) }; diff --git a/frontend/src/app/components/playlist/epg_source_selector.rs b/frontend/src/app/components/playlist/epg_source_selector.rs index 2a8c22595..abee05582 100644 --- a/frontend/src/app/components/playlist/epg_source_selector.rs +++ b/frontend/src/app/components/playlist/epg_source_selector.rs @@ -138,7 +138,7 @@ pub fn EpgSourceSelector(props: &EpgSourceSelectorProps) -> Html { }; if let Some((name, id)) = result { html! { - } } else { diff --git a/frontend/src/app/components/playlist/input_table.rs b/frontend/src/app/components/playlist/input_table.rs index e50d99c88..883422cf2 100644 --- a/frontend/src/app/components/playlist/input_table.rs +++ b/frontend/src/app/components/playlist/input_table.rs @@ -114,7 +114,7 @@ pub fn InputTable(props: &InputTableProps) -> Html { 1 => html! { }, - 2 => html! { dto.name.as_str() }, + 2 => html! { dto.name.as_ref() }, 3 => html! { }, 4 => html! { if matches!(dto.input_type, InputType::XtreamBatch | InputType::M3uBatch) { @@ -165,7 +165,7 @@ pub fn InputTable(props: &InputTableProps) -> Html { }, - 2 => html! { alias.name.as_str() }, + 2 => html! { alias.name.as_ref() }, 4 => html! { alias.url.as_str() }, 5 => alias.username.as_ref().map_or_else(|| html!{}, |u| html!{u}), 6 => alias.password.as_ref().map_or_else(|| html!{}, |pwd| html! { }), @@ -210,7 +210,7 @@ pub fn InputTable(props: &InputTableProps) -> Html { if result == DialogResult::Ok { if let (Some(on_delete), Some(selected)) = (on_delete, &*selected_dto) { if let InputRow::Input(dto) = &**selected { - on_delete.emit(dto.name.clone()); + on_delete.emit(dto.name.to_string()); } } } diff --git a/frontend/src/app/components/playlist/playlist_explorer.rs b/frontend/src/app/components/playlist/playlist_explorer.rs index 68f7d010f..06d32b255 100644 --- a/frontend/src/app/components/playlist/playlist_explorer.rs +++ b/frontend/src/app/components/playlist/playlist_explorer.rs @@ -1,25 +1,34 @@ +use crate::app::components::menu_item::MenuItem; +use crate::app::components::popup_menu::PopupMenu; +use crate::app::components::{AppIcon, IconButton, NoContent, Panel, Search}; +use crate::app::context::PlaylistExplorerContext; +use crate::hooks::use_service_context; +use crate::html_if; +use crate::model::{BusyStatus, EventMessage}; +use crate::services::DialogService; +use shared::error::{info_err_res, TuliproxError}; +use shared::model::{CommonPlaylistItem, PlaylistRequest, SearchRequest, SeriesStreamDetailEpisodeProperties, SeriesStreamProperties, StreamProperties, UiPlaylistGroup, VirtualId, XtreamCluster}; +use std::collections::HashMap; use std::fmt::Display; use std::rc::Rc; use std::str::FromStr; use wasm_bindgen::JsCast; use yew::platform::spawn_local; -use crate::app::context::PlaylistExplorerContext; use yew::prelude::*; use yew_hooks::use_clipboard; use yew_i18n::use_translation; -use shared::error::{TuliproxError, info_err_res}; -use shared::model::{CommonPlaylistItem, PlaylistRequest, SearchRequest, UiPlaylistGroup, XtreamCluster}; -use crate::app::components::{AppIcon, IconButton, NoContent, Panel, Search}; -use crate::app::components::menu_item::MenuItem; -use crate::app::components::popup_menu::PopupMenu; -use crate::hooks::use_service_context; -use crate::html_if; -use crate::model::{BusyStatus, EventMessage}; -use crate::services::DialogService; const COPY_LINK_TULIPROX_VIRTUAL_ID: &str = "copy_link_tuliprox_virtual_id"; const COPY_LINK_TULIPROX_WEBPLAYER_URL: &str = "copy_link_tuliprox_webplayer_url"; const COPY_LINK_PROVIDER_URL: &str = "copy_link_provider_url"; +const BACKGROUND_GRADIENT: &str = "linear-gradient(to right, rgba(0, 0, 0, 0.90) 30%, rgba(0, 0, 0, 0.75) 50%, rgba(0, 0, 0, 0.55) 60%, rgba(0, 0, 0, 0.35) 70%, rgba(0, 0, 0, 0.0) 85%)"; + +#[derive(Clone)] +struct ChannelSelection { + virtual_id: VirtualId, + cluster: XtreamCluster, + url: String, +} #[allow(clippy::enum_variant_names)] #[derive(Debug, Clone, Eq, PartialEq)] @@ -58,6 +67,7 @@ impl FromStr for ExplorerAction { enum ExplorerLevel { Categories, Group(Rc), + SeriesInfo(Rc, Rc, Option>), } #[function_component] @@ -68,7 +78,7 @@ pub fn PlaylistExplorer() -> Html { let service_ctx = use_service_context(); let current_item = use_state(|| ExplorerLevel::Categories); let playlist = use_state(|| (*context.playlist).clone()); - let selected_channel = use_state(|| None::>); + let selected_channel = use_state(|| None::); let popup_anchor_ref = use_state(|| None::); let popup_is_open = use_state(|| false); let clipboard = use_clipboard(); @@ -76,14 +86,13 @@ pub fn PlaylistExplorer() -> Html { let handle_cluster_change = { let cluster_vis = cluster_visible.clone(); - Callback::from(move |(name, _event) : (String, MouseEvent)| { + Callback::from(move |(name, _event): (String, MouseEvent)| { if let Ok(xc) = XtreamCluster::from_str(name.as_str()) { cluster_vis.set(xc); } }) }; - let handle_popup_close = { let set_is_open = popup_is_open.clone(); Callback::from(move |()| { @@ -96,14 +105,80 @@ pub fn PlaylistExplorer() -> Html { let set_anchor_ref = popup_anchor_ref.clone(); let set_is_open = popup_is_open.clone(); Callback::from(move |(dto, event): (Rc, MouseEvent)| { + event.prevent_default(); + event.stop_propagation(); if let Some(target) = event.target_dyn_into::() { - set_selected_channel.set(Some(dto.clone())); + set_selected_channel.set(Some(ChannelSelection { + virtual_id: dto.virtual_id, + cluster: dto.xtream_cluster.unwrap_or_default(), + url: dto.url.to_string(), + })); set_anchor_ref.set(Some(target)); set_is_open.set(true); } }) }; + let handle_episode_popup_onclick = { + let set_selected_channel = selected_channel.clone(); + let set_anchor_ref = popup_anchor_ref.clone(); + let set_is_open = popup_is_open.clone(); + Callback::from(move |(dto, event): (ChannelSelection, MouseEvent)| { + event.prevent_default(); + event.stop_propagation(); + if let Some(target) = event.target_dyn_into::() { + set_selected_channel.set(Some(dto)); + set_anchor_ref.set(Some(target)); + set_is_open.set(true); + } + }) + }; + + let load_series_info = { + let set_current_item = current_item.clone(); + let services = service_ctx.clone(); + let ctx = context.clone(); + + move |group: Rc, dto: Rc| { + if let Some(StreamProperties::Series(series_props)) = dto.additional_properties.as_ref() { + if series_props.details.as_ref().is_none_or(|d| d.episodes.as_ref().is_none_or(|e| e.is_empty())) { + let set_current_item = set_current_item.clone(); + let services = services.clone(); + let ctx = ctx.clone(); + services.event.broadcast(EventMessage::Busy(BusyStatus::Show)); + spawn_local(async move { + let mut handled = false; + if let Some(playlist_request) = ctx.playlist_request.as_ref() { + if let Some(props) = services.playlist.get_series_info(&dto, playlist_request).await { + handled = true; + set_current_item.set(ExplorerLevel::SeriesInfo(group.clone(), dto.clone(), Some(Box::new(props)))); + } + } + if !handled { + set_current_item.set(ExplorerLevel::SeriesInfo(group, dto, None)); + } + services.event.broadcast(EventMessage::Busy(BusyStatus::Hide)); + }); + } else { + set_current_item.set(ExplorerLevel::SeriesInfo(group, dto, None)); + }; + } else { + set_current_item.set(ExplorerLevel::SeriesInfo(group, dto, None)); + } + } + }; + + let handle_series_onclick = { + let set_current_item = current_item.clone(); + Callback::from(move |(dto, event): (Rc, MouseEvent)| { + event.prevent_default(); + event.stop_propagation(); + if let ExplorerLevel::Group(ref group) = *set_current_item { + load_series_info(group.clone(), dto.clone()); + } + }) + }; + { let set_playlist = playlist.clone(); let set_current_item = current_item.clone(); @@ -159,7 +234,7 @@ pub fn PlaylistExplorer() -> Html { let copy_to_clipboard = copy_to_clipboard.clone(); let services = services.clone(); let virtual_id = dto.virtual_id; - let cluster = dto.xtream_cluster.unwrap_or_default(); + let cluster = dto.cluster; let translate_clone = translate_clone.clone(); let target_id = *target_id; spawn_local(async move { @@ -180,7 +255,7 @@ pub fn PlaylistExplorer() -> Html { } ExplorerAction::CopyLinkProviderUrl => { if let Some(dto) = &*selected_channel { - copy_to_clipboard.emit(dto.url.clone()); + copy_to_clipboard.emit(dto.url.to_string()); } } } @@ -197,6 +272,9 @@ pub fn PlaylistExplorer() -> Html { ExplorerLevel::Group(_) => { current_item.set(ExplorerLevel::Categories); } + ExplorerLevel::SeriesInfo(ref group, _, _) => { + current_item.set(ExplorerLevel::Group(group.clone())); + } } }) }; @@ -305,11 +383,11 @@ pub fn PlaylistExplorer() -> Html { }; let render_channel_logo = |chan: &Rc| { - let logo = if chan.logo.is_empty() { chan.logo_small.as_str() } else { chan.logo.as_str() }; + let logo = if chan.logo.is_empty() { &chan.logo_small } else { &chan.logo }; if logo.is_empty() { html! {} } else { - html! { () { @@ -329,8 +407,15 @@ pub fn PlaylistExplorer() -> Html { group.channels.iter().map(|chan| { let chan_clone = chan.clone(); let popup_onclick = handle_popup_onclick.clone(); + let chan_click = if group.xtream_cluster == XtreamCluster::Series { + let chan_clone = chan.clone(); + let series_click = handle_series_onclick.clone(); + Some(Callback::from(move |event: MouseEvent| series_click.emit((chan_clone.clone(), event)))) + } else { + None + }; html! { - + + {e.title.clone()} + + + }}) + } +
+ + } + ) + } + } else { + Html::default() + }; + + html! { +
+
+ { series_html } +
+
+ {episodes_html} +
+
+ } + }; + html! {
@@ -356,6 +565,7 @@ pub fn PlaylistExplorer() -> Html { match *current_item { ExplorerLevel::Categories => html!{} , ExplorerLevel::Group(ref group) => html!{ {&group.title} }, + ExplorerLevel::SeriesInfo(_, ref pli, _) => html!{ {&pli.title} }, } }
@@ -369,6 +579,7 @@ pub fn PlaylistExplorer() -> Html { match *current_item { ExplorerLevel::Categories => html!{render_categories()} , ExplorerLevel::Group(ref group) => html!{ render_group(group) }, + ExplorerLevel::SeriesInfo(_, ref pli, ref props) => html!{ render_series_info(pli, props.as_ref()) }, } }
diff --git a/frontend/src/app/components/playlist/playlist_source_selector.rs b/frontend/src/app/components/playlist/playlist_source_selector.rs index 6ff5bb5e6..0ad9430a4 100644 --- a/frontend/src/app/components/playlist/playlist_source_selector.rs +++ b/frontend/src/app/components/playlist/playlist_source_selector.rs @@ -204,7 +204,7 @@ pub fn PlaylistSourceSelector(props: &PlaylistSourceSelectorProps) -> Html { }; if let Some((name, id)) = result { html! { - } } else { diff --git a/frontend/src/app/components/reveal_content.rs b/frontend/src/app/components/reveal_content.rs index 3e46f5a6d..e73344c07 100644 --- a/frontend/src/app/components/reveal_content.rs +++ b/frontend/src/app/components/reveal_content.rs @@ -25,6 +25,7 @@ pub fn RevealContent(props: &RevealContentProps) -> Html { let actions = props.actions.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); let content = content.clone(); let actions = actions.clone(); let dlg = dialog.clone(); diff --git a/frontend/src/app/components/source_editor/alias_item_form.rs b/frontend/src/app/components/source_editor/alias_item_form.rs index 3f50574dc..286cf0e73 100644 --- a/frontend/src/app/components/source_editor/alias_item_form.rs +++ b/frontend/src/app/components/source_editor/alias_item_form.rs @@ -1,8 +1,10 @@ +use std::sync::Arc; use crate::app::components::{Card, TextButton}; use crate::{edit_field_date, edit_field_number_i16, edit_field_number_u16, edit_field_text, edit_field_text_option, generate_form_reducer}; use shared::model::ConfigInputAliasDto; use yew::{function_component, html, use_reducer, Callback, Html, Properties, UseReducerHandle}; use yew_i18n::use_translation; +use shared::utils::Internable; const LABEL_ALIAS_NAME: &str = "LABEL.ALIAS_NAME"; const LABEL_URL: &str = "LABEL.URL"; @@ -16,7 +18,7 @@ generate_form_reducer!( state: AliasFormState { form: ConfigInputAliasDto }, action_name: AliasFormAction, fields { - Name => name: String, + Name => name: Arc, Url => url: String, Username => username: Option, Password => password: Option, @@ -42,7 +44,7 @@ pub fn AliasItemForm(props: &AliasItemFormProps) -> Html { AliasFormState { form: props.initial.clone().unwrap_or_else(|| ConfigInputAliasDto { id: 0, - name: String::new(), + name: "".intern(), url: String::new(), username: None, password: None, diff --git a/frontend/src/app/components/source_editor/block_view.rs b/frontend/src/app/components/source_editor/block_view.rs index 830114562..fc5a27596 100644 --- a/frontend/src/app/components/source_editor/block_view.rs +++ b/frontend/src/app/components/source_editor/block_view.rs @@ -69,17 +69,17 @@ pub fn BlockView(props: &BlockProps) -> Html { let (dto_title, show_type, is_batch) = match &block.instance { BlockInstance::Input(dto) => { dto.aliases.as_ref().map_or( - (dto.name.clone(), true, false), + (dto.name.to_string(), true, false), |a| { if a.is_empty() { - (dto.name.clone(), true, false) + (dto.name.to_string(), true, false) } else { - (if dto.name.is_empty() {a[0].name.clone()} else {dto.name.clone()}, true, true) + (if dto.name.is_empty() {a[0].name.to_string()} else {dto.name.to_string()}, true, true) } } ) }, - BlockInstance::Target(dto) => (dto.name.clone(), true, false), + BlockInstance::Target(dto) => (dto.name.to_string(), true, false), BlockInstance::Output(_output) => { (translate.t(&format!("SOURCE_EDITOR.BRICK_{}", block_type)), false, false) } @@ -128,6 +128,7 @@ pub fn BlockView(props: &BlockProps) -> Html { let on_connection_drop = props.on_connection_drop.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); on_connection_drop.emit(to_id) }) }} /> @@ -141,6 +142,7 @@ pub fn BlockView(props: &BlockProps) -> Html { let on_connection_start = props.on_connection_start.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); on_connection_start.emit(from_id); }) }} /> diff --git a/frontend/src/app/components/source_editor/editor_view.rs b/frontend/src/app/components/source_editor/editor_view.rs index 4109a42ce..ff2c2d89a 100644 --- a/frontend/src/app/components/source_editor/editor_view.rs +++ b/frontend/src/app/components/source_editor/editor_view.rs @@ -511,7 +511,10 @@ pub fn SourceEditor() -> Html { }) }; - let handle_drag_over = Callback::from(|e: DragEvent| e.prevent_default()); + let handle_drag_over = Callback::from(|e: DragEvent| { + e.prevent_default(); + e.stop_propagation(); + }); let handle_drag_end = { let cursor_grabbing = cursor_grabbing.clone(); Callback::from(move |e: DragEvent| { diff --git a/frontend/src/app/components/source_editor/input_form.rs b/frontend/src/app/components/source_editor/input_form.rs index d4642a54d..eba0b6c29 100644 --- a/frontend/src/app/components/source_editor/input_form.rs +++ b/frontend/src/app/components/source_editor/input_form.rs @@ -7,9 +7,12 @@ use shared::model::{ConfigInputAliasDto, ConfigInputDto, ConfigInputOptionsDto, use std::collections::HashMap; use std::fmt::Display; use std::rc::Rc; +use std::str::FromStr; use web_sys::MouseEvent; -use yew::{classes, function_component, html, use_context, use_effect_with, use_memo, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; +use yew::{function_component, html, use_context, use_effect_with, use_memo, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; use yew_i18n::use_translation; +use shared::error::TuliproxError; +use shared::info_err_res; const LABEL_NAME: &str = "LABEL.NAME"; const LABEL_INPUT_TYPE: &str = "LABEL.INPUT_TYPE"; @@ -35,6 +38,12 @@ const LABEL_XTREAM_LIVE_STREAM_USE_PREFIX: &str = "LABEL.LIVE_STREAM_USE_PREFIX" const LABEL_XTREAM_LIVE_STREAM_WITHOUT_EXTENSION: &str = "LABEL.LIVE_STREAM_WITHOUT_EXTENSION"; const LABEL_CACHE_DURATION: &str = "LABEL.CACHE_DURATION"; +const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; +const LABEL_OPTIONS: &str = "LABEL.OPTIONS"; +const LABEL_STAGED: &str = "LABEL.STAGED"; +const LABEL_ADVANCED: &str = "LABEL.ADVANCED"; +const LABEL_ALIAS: &str = "LABEL.ALIAS"; + #[derive(Copy, Clone, PartialEq, Eq)] enum InputFormPage { @@ -42,15 +51,40 @@ enum InputFormPage { Options, Staged, Advanced, + Alias +} + +impl InputFormPage { + const MAIN: &str = "Main"; + const OPTIONS: &str = "Options"; + const STAGED: &str = "Staged"; + const ADVANCED: &str = "Advanced"; + const ALIAS: &str = "Alias"; +} + +impl FromStr for InputFormPage { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s { + Self::MAIN => Ok(InputFormPage::Main), + Self::OPTIONS => Ok(InputFormPage::Options), + Self::STAGED => Ok(InputFormPage::Staged), + Self::ADVANCED => Ok(InputFormPage::Advanced), + Self::ALIAS => Ok(InputFormPage::Alias), + _ => info_err_res!("Unknown input form page: {s}"), + } + } } impl Display for InputFormPage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", match *self { - InputFormPage::Main => "Main", - InputFormPage::Options => "Options", - InputFormPage::Staged => "Staged", - InputFormPage::Advanced => "Advanced", + InputFormPage::Main => Self::MAIN, + InputFormPage::Options => Self::OPTIONS, + InputFormPage::Staged => Self::STAGED, + InputFormPage::Advanced => Self::ADVANCED, + InputFormPage::Alias => Self::ALIAS, }) } } @@ -119,11 +153,20 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { .map(ToString::to_string) .collect::>() }); - let view_visible = use_state(|| InputFormPage::Main.to_string()); + let view_visible = use_state(|| InputFormPage::Main); - let on_tab_click = { - let view_visible = view_visible.clone(); - Callback::from(move |page: InputFormPage| view_visible.set(page.to_string())) + // let on_tab_click = { + // let view_visible = view_visible.clone(); + // Callback::from(move |page: InputFormPage| view_visible.set(page)) + // }; + + let handle_menu_click = { + let active_menu = view_visible.clone(); + Callback::from(move |(name, _): (String, _)| { + if let Ok(view_type) = InputFormPage::from_str(&name) { + active_menu.set(view_type); + } + }) }; let input_form_state: UseReducerHandle = @@ -150,6 +193,7 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { // State for showing item forms let show_epg_form_state = use_state(|| false); let show_alias_form_state = use_state(|| false); + let edit_alias = use_state(|| None::); let staged_input_types = use_memo(staged_input_state.form.input_type, |input_type| { let default_it = input_type; @@ -237,9 +281,21 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { let handle_add_alias_item = { let aliases = aliases_state.clone(); let show_alias_form = show_alias_form_state.clone(); + let edit_alias = edit_alias.clone(); Callback::from(move |alias: ConfigInputAliasDto| { let mut items = (*aliases).clone(); - items.push(alias); + if let Some(e) = edit_alias.as_ref() { + if let Some(pos) = items.iter().position(|x| x.name == e.name) { + if let Some(slot) = items.get_mut(pos) { + *slot = alias; + } + } else { + items.push(alias); + } + edit_alias.set(None); + } else { + items.push(alias); + } aliases.set(items); show_alias_form.set(false); }) @@ -247,15 +303,19 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { let handle_close_add_alias_item = { let show_alias_form = show_alias_form_state.clone(); + let edit_alias = edit_alias.clone(); Callback::from(move |()| { show_alias_form.set(false); + edit_alias.set(None); }) }; let handle_show_add_alias_item = { let show_alias_form = show_alias_form_state.clone(); + let edit_alias = edit_alias.clone(); Callback::from(move |_| { show_alias_form.set(true); + edit_alias.set(None); }) }; @@ -264,6 +324,7 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { let alias_list = aliases_state.clone(); Callback::from(move |(idx, e): (String, MouseEvent)| { e.prevent_default(); + e.stop_propagation(); if let Ok(index) = idx.parse::() { let mut items = (*alias_list).clone(); if index < items.len() { @@ -274,10 +335,30 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { }) }; + let handle_edit_alias_list_item = { + let alias_list = aliases_state.clone(); + let show_alias_form = show_alias_form_state.clone(); + let edit_alias = edit_alias.clone(); + + Callback::from(move |(idx, e): (String, MouseEvent)| { + e.prevent_default(); + e.stop_propagation(); + if let Ok(index) = idx.parse::() { + let items = (*alias_list).clone(); + if index < items.len() { + let item = items.get(index).cloned(); + edit_alias.set(item); + show_alias_form.set(true); + } + } + }) + }; + let handle_remove_epg_source = { let epg_list = epg_sources_state.clone(); Callback::from(move |(idx, e): (String, MouseEvent)| { e.prevent_default(); + e.stop_propagation(); if let Ok(index) = idx.parse::() { let mut items = (*epg_list).clone(); if index < items.len() { @@ -361,27 +442,19 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { let render_input = || { let input_method_selection = Rc::new(vec![input_form_state.form.method.to_string()]); let input_form_state_disp = input_form_state.clone(); - let aliases = aliases_state.clone(); - let show_alias_form = show_alias_form_state.clone(); html! { - if *show_alias_form { - - } else { -
+
{ edit_field_text!(input_form_state, translate.t(LABEL_NAME), name, ConfigInputFormAction::Name) } { edit_field_bool!(input_form_state, translate.t(LABEL_ENABLED), enabled, ConfigInputFormAction::Enabled) } -
+
{ edit_field_text!(input_form_state, translate.t(LABEL_URL), url, ConfigInputFormAction::Url) } -
+
{ edit_field_text_option!(input_form_state, translate.t(LABEL_USERNAME), username, ConfigInputFormAction::Username) } { edit_field_text_option!(input_form_state, translate.t(LABEL_PASSWORD), password, ConfigInputFormAction::Password, true) } - { edit_field_number_u16!(input_form_state, translate.t(LABEL_MAX_CONNECTIONS), max_connections, ConfigInputFormAction::MaxConnections) } - { edit_field_number_i16!(input_form_state, translate.t(LABEL_PRIORITY), priority, ConfigInputFormAction::Priority) } + { edit_field_number_u16!(input_form_state, translate.t(LABEL_MAX_CONNECTIONS), max_connections, ConfigInputFormAction::MaxConnections) } + { edit_field_number_i16!(input_form_state, translate.t(LABEL_PRIORITY), priority, ConfigInputFormAction::Priority) } { edit_field_date!(input_form_state, translate.t(LABEL_EXP_DATE), exp_date, ConfigInputFormAction::ExpDate) } { edit_field_text_option!(input_form_state, translate.t(LABEL_CACHE_DURATION), cache_duration, ConfigInputFormAction::CacheDuration) } { config_field_child!(translate.t(LABEL_FETCH_METHOD), { @@ -397,9 +470,26 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { selected={input_method_selection} /> }})} -
+
{ edit_field_text_option!(input_form_state, translate.t(LABEL_PERSIST), persist, ConfigInputFormAction::Persist) } - // Aliases Section + + } + }; + + let render_alias = || { + let aliases = aliases_state.clone(); + let show_alias_form = show_alias_form_state.clone(); + let edit_alias = edit_alias.clone(); + + html! { + + if *show_alias_form { + + } else { { config_field_child!(translate.t(LABEL_ALIASES), { let aliases_list = aliases.clone(); html! { @@ -409,10 +499,16 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { for (*aliases_list).iter().enumerate().map(|(idx, alias)| { html! {
+
+ +
{&alias.name}{" - "}{&alias.url}
@@ -421,17 +517,19 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { }) }
- - +
+ +
+ } })} - } + } } }; @@ -513,7 +611,7 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { let headers_state = headers_state.clone(); let epg_sources_state = epg_sources_state.clone(); let aliases_state = aliases_state.clone(); - + Callback::from(move |_| { let mut input = input_form_state.data().clone(); @@ -580,37 +678,39 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { let render_edit_mode = || { html! { -
-
- { - for [ - InputFormPage::Main, - InputFormPage::Options, - InputFormPage::Staged, - InputFormPage::Advanced - ].iter().map(|page| { - let page_str = page.to_string(); - let active = *view_visible == page_str; - let on_tab_click = { - let on_tab_click = on_tab_click.clone(); - let page = *page; - Callback::from(move |_| on_tab_click.emit(page)) - }; - html! { - - } - }) - } -
-
+
+ //
+ // { + // for [ + // InputFormPage::Main, + // InputFormPage::Options, + // InputFormPage::Staged, + // InputFormPage::Advanced + // ].iter().map(|page| { + // let active = &*view_visible == page; + // let on_tab_click = { + // let on_tab_click = on_tab_click.clone(); + // let page = *page; + // Callback::from(move |_| on_tab_click.emit(page)) + // }; + // html! { + // + // } + // }) + // } + //
+
{render_input()} + + {render_alias()} + {render_options()} @@ -625,6 +725,18 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { } }; + let render_sidebar = || { + html! { +
+ + + + + +
+ } + }; + html! {
@@ -637,7 +749,10 @@ pub fn ConfigInputView(props: &ConfigInputViewProps) -> Html { title={ translate.t("LABEL.OK")} onclick={handle_apply_input}>
+
+ { render_sidebar() } { render_edit_mode() }
+
} } diff --git a/frontend/src/app/components/source_editor/output_strm_form.rs b/frontend/src/app/components/source_editor/output_strm_form.rs index 17f7d0c4d..b76126068 100644 --- a/frontend/src/app/components/source_editor/output_strm_form.rs +++ b/frontend/src/app/components/source_editor/output_strm_form.rs @@ -1,12 +1,15 @@ use crate::app::components::config::HasFormData; use crate::app::components::select::Select; -use crate::app::components::{BlockId, BlockInstance, Card, DropDownOption, DropDownSelection, EditMode, FilterInput, Panel, SourceEditorContext, TextButton}; +use crate::app::components::{BlockId, BlockInstance, Card, DropDownOption, DropDownSelection, EditMode, FilterInput, IconButton, Panel, SourceEditorContext, TextButton}; use crate::{config_field_child, edit_field_bool, edit_field_list_option, edit_field_text, edit_field_text_option, generate_form_reducer}; use shared::model::{StrmExportStyle, StrmTargetOutputDto, TargetOutputDto}; use std::fmt::Display; use std::rc::Rc; -use yew::{classes, function_component, html, use_context, use_effect_with, use_memo, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; +use std::str::FromStr; +use yew::{function_component, html, use_context, use_effect_with, use_memo, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; use yew_i18n::use_translation; +use shared::error::TuliproxError; +use shared::info_err_res; const LABEL_DIRECTORY: &str = "LABEL.DIRECTORY"; const LABEL_USERNAME: &str = "LABEL.USERNAME"; @@ -18,6 +21,8 @@ const LABEL_STRM_PROPS: &str = "LABEL.STRM_PROPS"; const LABEL_FILTER: &str = "LABEL.FILTER"; const LABEL_ADD_QUALITY_TO_FILENAME: &str = "LABEL.ADD_QUALITY_TO_FILENAME"; const LABEL_ADD_PROPERTY: &str = "LABEL.ADD_PROPERTY"; +const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; +const LABEL_OPTIONS: &str = "LABEL.OPTIONS"; #[derive(Copy, Clone, PartialEq, Eq)] enum StrmFormPage { @@ -25,11 +30,30 @@ enum StrmFormPage { Options, } + +impl StrmFormPage { + const MAIN: &str = "Main"; + const OPTIONS: &str = "Options"; +} + +impl FromStr for StrmFormPage { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s { + Self::MAIN => Ok(StrmFormPage::Main), + Self::OPTIONS => Ok(StrmFormPage::Options), + _ => info_err_res!("Unknown strm output form page: {s}"), + } + } +} + + impl Display for StrmFormPage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", match *self { - StrmFormPage::Main => "Main", - StrmFormPage::Options => "Options", + StrmFormPage::Main => Self::MAIN, + StrmFormPage::Options => Self::OPTIONS, }) } } @@ -67,11 +91,15 @@ pub fn StrmTargetOutputView(props: &StrmTargetOutputViewProps) -> Html { modified: false, }); - let view_visible = use_state(|| StrmFormPage::Main.to_string()); + let view_visible = use_state(|| StrmFormPage::Main); - let on_tab_click = { - let view_visible = view_visible.clone(); - Callback::from(move |page: StrmFormPage| view_visible.set(page.to_string())) + let handle_menu_click = { + let active_menu = view_visible.clone(); + Callback::from(move |(name, _): (String, _)| { + if let Ok(view_type) = StrmFormPage::from_str(&name) { + active_menu.set(view_type); + } + }) }; let export_styles = use_memo(output_form_state.form.style, |style| { @@ -160,32 +188,8 @@ pub fn StrmTargetOutputView(props: &StrmTargetOutputViewProps) -> Html { let render_edit_mode = || { html! { -
-
- { - for [ - StrmFormPage::Main, - StrmFormPage::Options, - ].iter().map(|page| { - let page_str = page.to_string(); - let active = *view_visible == page_str; - let on_tab_click = { - let on_tab_click = on_tab_click.clone(); - let page = *page; - Callback::from(move |_| on_tab_click.emit(page)) - }; - html! { - - } - }) - } -
-
+
+
{render_main()} @@ -197,6 +201,15 @@ pub fn StrmTargetOutputView(props: &StrmTargetOutputViewProps) -> Html { } }; + let render_sidebar = || { + html! { +
+ + +
+ } + }; + let handle_apply = { let source_editor_ctx = source_editor_ctx.clone(); let output_form_state = output_form_state.clone(); @@ -227,7 +240,10 @@ pub fn StrmTargetOutputView(props: &StrmTargetOutputViewProps) -> Html { title={ translate.t("LABEL.OK")} onclick={handle_apply}>
- { render_edit_mode() } +
+ { render_sidebar() } + { render_edit_mode() } +
} } diff --git a/frontend/src/app/components/source_editor/output_xtream_form.rs b/frontend/src/app/components/source_editor/output_xtream_form.rs index ab6c198ef..a10bed11b 100644 --- a/frontend/src/app/components/source_editor/output_xtream_form.rs +++ b/frontend/src/app/components/source_editor/output_xtream_form.rs @@ -4,9 +4,12 @@ use crate::{config_field_child, edit_field_bool, edit_field_number_u16, edit_fie use shared::model::{TargetOutputDto, TraktApiConfigDto, TraktConfigDto, TraktContentType, TraktListConfigDto, XtreamTargetOutputDto}; use std::fmt::Display; use std::rc::Rc; +use std::str::FromStr; use web_sys::MouseEvent; -use yew::{classes, function_component, html, use_context, use_effect_with, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; +use yew::{function_component, html, use_context, use_effect_with, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; use yew_i18n::use_translation; +use shared::error::TuliproxError; +use shared::info_err_res; const LABEL_SKIP_DIRECT_SOURCE: &str = "LABEL.SKIP_DIRECT_SOURCE"; const LABEL_LIVE: &str = "LABEL.LIVE"; @@ -22,21 +25,50 @@ const LABEL_TRAKT_LISTS: &str = "LABEL.TRAKT_LISTS"; const LABEL_ADD_TRAKT_LIST: &str = "LABEL.ADD_TRAKT_LIST"; const LABEL_API_CONFIGURATION: &str = "LABEL.API_CONFIGURATION"; const LABEL_USER_AGENT: &str = "LABEL.API_USER_AGENT"; +const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; +const LABEL_TRAKT: &str = "LABEL.TRAKT"; +const LABEL_ENABLED: &str = "LABEL.ENABLED"; #[derive(Copy, Clone, PartialEq, Eq)] -enum OutputFormPage { +enum XtreamOutputFormPage { Main, Trakt, } -impl Display for OutputFormPage { +impl XtreamOutputFormPage { + const MAIN: &str = "Main"; + const TRAKT: &str = "Trakt"; +} + +impl FromStr for XtreamOutputFormPage { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s { + Self::MAIN => Ok(XtreamOutputFormPage::Main), + Self::TRAKT => Ok(XtreamOutputFormPage::Trakt), + _ => info_err_res!("Unknown xtream output form page: {s}"), + } + } +} + +impl Display for XtreamOutputFormPage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", match *self { - OutputFormPage::Main => "Main", - OutputFormPage::Trakt => "Trakt", + XtreamOutputFormPage::Main => Self::MAIN, + XtreamOutputFormPage::Trakt => Self::TRAKT, }) } } + +generate_form_reducer!( + state: TraktConfigFormState { form: TraktConfigDto }, + action_name: TraktConfigFormAction, + fields { + Enabled => enabled: bool, + } +); + generate_form_reducer!( state: TraktApiConfigFormState { form: TraktApiConfigDto }, action_name: TraktApiConfigFormAction, @@ -80,6 +112,12 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { modified: false, }); + let trakt_state: UseReducerHandle = + use_reducer(|| TraktConfigFormState { + form: TraktConfigDto::default(), + modified: false, + }); + let trakt_api_state: UseReducerHandle = use_reducer(|| TraktApiConfigFormState { form: TraktApiConfigDto::default(), @@ -92,15 +130,20 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { // State for showing trakt list form let show_trakt_list_form_state = use_state(|| false); - let view_visible = use_state(|| OutputFormPage::Main.to_string()); + let view_visible = use_state(|| XtreamOutputFormPage::Main); - let on_tab_click = { - let view_visible = view_visible.clone(); - Callback::from(move |page: OutputFormPage| view_visible.set(page.to_string())) + let handle_menu_click = { + let active_menu = view_visible.clone(); + Callback::from(move |(name, _): (String, _)| { + if let Ok(view_type) = XtreamOutputFormPage::from_str(&name) { + active_menu.set(view_type); + } + }) }; { let output_form_state = output_form_state.clone(); + let trakt_state = trakt_state.clone(); let trakt_api_state = trakt_api_state.clone(); let trakt_lists_state = trakt_lists_state.clone(); @@ -112,14 +155,17 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { // Load Trakt configuration if let Some(trakt) = &target.trakt { + trakt_state.dispatch(TraktConfigFormAction::SetAll(trakt.clone())); trakt_api_state.dispatch(TraktApiConfigFormAction::SetAll(trakt.api.clone())); trakt_lists_state.set(trakt.lists.clone()); } else { + trakt_state.dispatch(TraktConfigFormAction::SetAll(TraktConfigDto::default())); trakt_api_state.dispatch(TraktApiConfigFormAction::SetAll(TraktApiConfigDto::default())); trakt_lists_state.set(Vec::new()); } } else { output_form_state.dispatch(XtreamTargetOutputFormAction::SetAll(XtreamTargetOutputDto::default())); + trakt_state.dispatch(TraktConfigFormAction::SetAll(TraktConfigDto::default())); trakt_api_state.dispatch(TraktApiConfigFormAction::SetAll(TraktApiConfigDto::default())); trakt_lists_state.set(Vec::new()); } @@ -202,7 +248,8 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { let render_trakt = || { let trakt_lists = trakt_lists_state.clone(); - let trakt_api = trakt_api_state.clone(); + let trakt_form = trakt_state.clone(); + let trakt_api_form = trakt_api_state.clone(); let show_trakt_list_form = show_trakt_list_form_state.clone(); html! { @@ -214,12 +261,13 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { /> } else { // Trakt API Configuration + { edit_field_bool!(trakt_form, translate.t(LABEL_ENABLED), enabled, TraktConfigFormAction::Enabled) }

{translate.t(LABEL_API_CONFIGURATION)}

- { edit_field_text!(trakt_api, translate.t(LABEL_TRAKT_API_KEY), api_key, TraktApiConfigFormAction::ApiKey) } - { edit_field_text!(trakt_api, translate.t(LABEL_TRAKT_API_VERSION), version, TraktApiConfigFormAction::Version) } - { edit_field_text!(trakt_api, translate.t(LABEL_TRAKT_API_URL), url, TraktApiConfigFormAction::Url) } - { edit_field_text!(trakt_api, translate.t(LABEL_USER_AGENT), user_agent, TraktApiConfigFormAction::UserAgent) } + { edit_field_text!(trakt_api_form, translate.t(LABEL_TRAKT_API_KEY), api_key, TraktApiConfigFormAction::ApiKey) } + { edit_field_text!(trakt_api_form, translate.t(LABEL_TRAKT_API_VERSION), version, TraktApiConfigFormAction::Version) } + { edit_field_text!(trakt_api_form, translate.t(LABEL_TRAKT_API_URL), url, TraktApiConfigFormAction::Url) } + { edit_field_text!(trakt_api_form, translate.t(LABEL_USER_AGENT), user_agent, TraktApiConfigFormAction::UserAgent) }
// Trakt Lists @@ -279,35 +327,11 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { let render_edit_mode = || { html! {
-
- { - for [ - OutputFormPage::Main, - OutputFormPage::Trakt, - ].iter().map(|page| { - let page_str = page.to_string(); - let active = *view_visible == page_str; - let on_tab_click = { - let on_tab_click = on_tab_click.clone(); - let page = *page; - Callback::from(move |_| on_tab_click.emit(page)) - }; - html! { - - } - }) - } -
- + {render_output()} - + {render_trakt()}
@@ -315,9 +339,29 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { } }; + let render_sidebar = || { + let main_class = format!( + "tp__app-sidebar-menu--{}{}", + XtreamOutputFormPage::Main, + if *view_visible == XtreamOutputFormPage::Main { " active" } else { "" } + ); + let trakt_class = format!( + "tp__app-sidebar-menu--{}{}", + XtreamOutputFormPage::Trakt, + if *view_visible == XtreamOutputFormPage::Trakt { " active" } else { "" } + ); + html! { +
+ + +
+ } + }; + let handle_apply_target = { let source_editor_ctx = source_editor_ctx.clone(); let output_form_state = output_form_state.clone(); + let trakt_state = trakt_state.clone(); let trakt_api_state = trakt_api_state.clone(); let trakt_lists_state = trakt_lists_state.clone(); let block_id = props.block_id; @@ -330,6 +374,7 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { None } else { Some(TraktConfigDto { + enabled: trakt_state.data().enabled, api: trakt_api_state.data().clone(), lists: trakt_lists, }) @@ -348,7 +393,7 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { html! {
-
+
Html { title={ translate.t("LABEL.OK")} onclick={handle_apply_target}>
- { render_edit_mode() } +
+ { render_sidebar() } + { render_edit_mode() } +
- } + } } diff --git a/frontend/src/app/components/source_editor/target_form.rs b/frontend/src/app/components/source_editor/target_form.rs index f3c783f15..354ac9510 100644 --- a/frontend/src/app/components/source_editor/target_form.rs +++ b/frontend/src/app/components/source_editor/target_form.rs @@ -1,12 +1,15 @@ use crate::app::components::config::HasFormData; use crate::app::components::select::Select; -use crate::app::components::{BlockId, BlockInstance, Card, ClusterFlagsInput, ClusterFlagsInputMode, DropDownOption, DropDownSelection, EditMode, FilterInput, Panel, SourceEditorContext, TextButton}; +use crate::app::components::{BlockId, BlockInstance, Card, ClusterFlagsInput, ClusterFlagsInputMode, DropDownOption, DropDownSelection, EditMode, FilterInput, IconButton, Panel, SourceEditorContext, TextButton}; use crate::{config_field_child, edit_field_bool, edit_field_list_option, edit_field_text, generate_form_reducer}; use shared::model::{ClusterFlags, ConfigTargetDto, ConfigTargetOptions, ProcessingOrder}; use std::fmt::Display; use std::rc::Rc; -use yew::{classes, function_component, html, use_context, use_effect_with, use_memo, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; +use std::str::FromStr; +use yew::{function_component, html, use_context, use_effect_with, use_memo, use_reducer, use_state, Callback, Html, Properties, UseReducerHandle}; use yew_i18n::use_translation; +use shared::error::TuliproxError; +use shared::info_err_res; const LABEL_ENABLED: &str = "LABEL.ENABLED"; const LABEL_NAME: &str = "LABEL.NAME"; @@ -21,6 +24,8 @@ const LABEL_IGNORE_LOGO: &str = "LABEL.IGNORE_LOGO"; const LABEL_SHARE_LIVE_STREAMS: &str = "LABEL.SHARE_LIVE_STREAMS"; const LABEL_REMOVE_DUPLICATES: &str = "LABEL.REMOVE_DUPLICATES"; const LABEL_FORCE_REDIRECT: &str = "LABEL.FORCE_REDIRECT"; +const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; +const LABEL_OPTIONS: &str = "LABEL.OPTIONS"; #[derive(Copy, Clone, PartialEq, Eq)] enum TargetFormPage { @@ -28,6 +33,23 @@ enum TargetFormPage { Options, } +impl TargetFormPage { + const MAIN: &str = "Main"; + const OPTIONS: &str = "Options"; +} + +impl FromStr for TargetFormPage { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s { + Self::MAIN => Ok(TargetFormPage::Main), + Self::OPTIONS => Ok(TargetFormPage::Options), + _ => info_err_res!("Unknown target form page: {s}"), + } + } +} + impl Display for TargetFormPage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match *self { @@ -89,13 +111,19 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { modified: false, }); - let view_visible = use_state(|| TargetFormPage::Main.to_string()); + let view_visible = use_state(|| TargetFormPage::Main); - let on_tab_click = { - let view_visible = view_visible.clone(); - Callback::from(move |page: TargetFormPage| view_visible.set(page.to_string())) + + let handle_menu_click = { + let active_menu = view_visible.clone(); + Callback::from(move |(name, _): (String, _)| { + if let Ok(view_type) = TargetFormPage::from_str(&name) { + active_menu.set(view_type); + } + }) }; + let processing_orders = use_memo(target_form_state.clone(), |target_state: &UseReducerHandle| { let default_po = target_state.form.processing_order; [ @@ -208,32 +236,8 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { let render_edit_mode = || { html! { -
-
- { - for [ - TargetFormPage::Main, - TargetFormPage::Options, - ].iter().map(|page| { - let page_str = page.to_string(); - let active = *view_visible == page_str; - let on_tab_click = { - let on_tab_click = on_tab_click.clone(); - let page = *page; - Callback::from(move |_| on_tab_click.emit(page)) - }; - html! { - - } - }) - } -
-
+
+
{render_target()} @@ -245,6 +249,15 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { } }; + let render_sidebar = || { + html! { +
+ + +
+ } + }; + let handle_apply_target = { let source_editor_ctx = source_editor_ctx.clone(); let target_form_state = target_form_state.clone(); @@ -281,7 +294,10 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { title={ translate.t("LABEL.OK")} onclick={handle_apply_target}>
- { render_edit_mode() } +
+ { render_sidebar() } + { render_edit_mode() } +
} } diff --git a/frontend/src/app/components/tag_list.rs b/frontend/src/app/components/tag_list.rs index 159a599f5..6675024c0 100644 --- a/frontend/src/app/components/tag_list.rs +++ b/frontend/src/app/components/tag_list.rs @@ -70,6 +70,7 @@ pub fn TagList(props: &TagListProps) -> Html { Callback::from(move |e: KeyboardEvent| { if e.key() == "Enter" { e.prevent_default(); + e.stop_propagation(); let val = (*new_tag).trim().to_string(); if !val.is_empty() && !tag_state.iter().any(|t| t.label == val) { let mut updated = (*tag_state).clone(); diff --git a/frontend/src/app/components/text_button.rs b/frontend/src/app/components/text_button.rs index c1596aa9d..b7d29454f 100644 --- a/frontend/src/app/components/text_button.rs +++ b/frontend/src/app/components/text_button.rs @@ -23,12 +23,16 @@ pub fn TextButton(props: &TextButtonProps) -> Html { let name = props.name.clone(); Callback::from(move |e: MouseEvent| { e.prevent_default(); + e.stop_propagation(); click.emit(name.clone()); }) }; html! { -