From 55191d6c84eeba9bf448352c22940ddf74119316 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 09:53:49 +0100 Subject: [PATCH 01/11] vod resolve refactored Arc is now Client --- .../src/api/endpoints/api_playlist_utils.rs | 6 +- backend/src/api/endpoints/hls_api.rs | 2 +- backend/src/api/endpoints/v1_api.rs | 2 +- backend/src/api/endpoints/v1_api_config.rs | 2 +- backend/src/api/endpoints/v1_api_playlist.rs | 11 +- backend/src/api/endpoints/xtream_api.rs | 6 +- backend/src/api/main_api.rs | 11 +- .../model/streams/provider_stream_factory.rs | 14 +-- backend/src/api/scheduler.rs | 9 +- backend/src/main.rs | 2 +- backend/src/messaging.rs | 13 +-- backend/src/model/xmltv.rs | 3 +- backend/src/processing/playlist_watch.rs | 5 +- backend/src/processing/processor/playlist.rs | 43 ++++--- backend/src/processing/processor/trakt.rs | 7 +- backend/src/processing/processor/xtream.rs | 3 +- .../src/processing/processor/xtream_series.rs | 7 +- .../src/processing/processor/xtream_vod.rs | 109 +++++++++--------- backend/src/utils/network/epg.rs | 9 +- backend/src/utils/network/ip_checker.rs | 8 +- backend/src/utils/network/m3u.rs | 2 +- backend/src/utils/network/request.rs | 29 +++-- backend/src/utils/network/xtream.rs | 20 ++-- backend/src/utils/telegram.rs | 3 +- backend/src/utils/trakt/client.rs | 5 +- 25 files changed, 163 insertions(+), 168 deletions(-) diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index 7658e5433..dabc8a398 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -141,13 +141,13 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Optio (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response() } -pub(in crate::api::endpoints) async fn get_playlist(client: Arc, cfg_input: Option<&Arc>, cfg: &Arc, accept: Option<&String>) -> impl IntoResponse + Send { +pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cfg_input: Option<&Arc>, cfg: &Arc, accept: Option<&String>) -> impl IntoResponse + Send { match cfg_input { Some(input) => { let (result, errors) = match input.input_type { - InputType::M3u | InputType::M3uBatch => m3u::get_m3u_playlist(&client, cfg, input, &cfg.working_dir).await, - InputType::Xtream | InputType::XtreamBatch => xtream::get_xtream_playlist(cfg, &client, input, &cfg.working_dir).await, + InputType::M3u | InputType::M3uBatch => m3u::get_m3u_playlist(client, cfg, input, &cfg.working_dir).await, + InputType::Xtream | InputType::XtreamBatch => xtream::get_xtream_playlist(cfg, client, input, &cfg.working_dir).await, }; if result.is_empty() { let error_strings: Vec = errors.iter().map(std::string::ToString::to_string).collect(); diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index c54e2148c..57f0fb53b 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -119,7 +119,7 @@ pub(in crate::api) async fn handle_hls_stream_request( let headers = request::get_request_headers(None, Some(&forwarded), disabled_headers.as_ref()); let input_source = InputSource::from(input).with_url(request_url); match request::download_text_content( - Arc::clone(&app_state.http_client.load()), + &app_state.http_client.load(), disabled_headers.as_ref(), &input_source, Some(&headers), diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index 523022c34..39ba81507 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -93,7 +93,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State { let reader = Cursor::new(content); let mut geoip = GeoIp::new(); diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 3bfb32e52..8a4a6c64f 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -123,7 +123,7 @@ async fn config_batch_content( .reverse_proxy .as_ref() .and_then(|r| r.disabled_header.clone()); - return match download_text_content(Arc::clone(&app_state.http_client.load()), disabled_headers.as_ref(), &input_source, None, None).await { + return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await { Ok((content, _path)) => { // Return CSV with explicit content-type try_unwrap_body!(axum::response::Response::builder() diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index 8ebbff432..db8f19ba7 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -60,14 +60,14 @@ async fn playlist_update( let process_targets = app_state.app_config.sources.load().validate_targets(user_targets.as_ref()); match process_targets { Ok(valid_targets) => { - let http_client = Arc::clone(&app_state.http_client.load()); + let http_client = app_state.http_client.load().as_ref().clone(); let app_config = Arc::clone(&app_state.app_config); let event_manager = Arc::clone(&app_state.event_manager); let playlist_state = Arc::clone(&app_state.playlists); let valid_targets = Arc::new(valid_targets); tokio::spawn({ async move { - playlist::exec_processing(http_client, app_config, valid_targets, Some(event_manager), Some(playlist_state)).await; + playlist::exec_processing(&http_client, app_config, valid_targets, Some(event_manager), Some(playlist_state)).await; } }); axum::http::StatusCode::ACCEPTED.into_response() @@ -86,18 +86,19 @@ async fn playlist_content( axum::extract::Json(playlist_req): axum::extract::Json, ) -> impl IntoResponse + Send { let config = app_state.app_config.config.load(); + let client = app_state.http_client.load(); match playlist_req { PlaylistRequest::Target(target_id) => { get_playlist_for_target(app_state.app_config.get_target_by_id(target_id).as_deref(), &app_state.app_config, accept.as_ref()).await.into_response() } PlaylistRequest::Input(input_id) => { - get_playlist(Arc::clone(&app_state.http_client.load()), app_state.app_config.get_input_by_id(input_id).as_ref(), &config, accept.as_ref()).await.into_response() + get_playlist(client.as_ref(), app_state.app_config.get_input_by_id(input_id).as_ref(), &config, accept.as_ref()).await.into_response() } PlaylistRequest::CustomXtream(xtream) => { match Url::parse(&xtream.url) { Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url)); - get_playlist(Arc::clone(&app_state.http_client.load()), Some(&input), &config, accept.as_ref()).await.into_response() + get_playlist(client.as_ref(), Some(&input), &config, accept.as_ref()).await.into_response() } _ => { (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response() @@ -108,7 +109,7 @@ async fn playlist_content( match Url::parse(&m3u.url) { Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { let input = Arc::new(create_config_input_for_m3u(&m3u.url)); - get_playlist(Arc::clone(&app_state.http_client.load()), Some(&input), &config, accept.as_ref()).await.into_response() + get_playlist(client.as_ref(), Some(&input), &config, accept.as_ref()).await.into_response() } _ => { (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response() diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index eea853b46..897f55d07 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -981,7 +981,7 @@ async fn xtream_get_stream_info_response( if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live { return redirect(&info_url).into_response(); } else if let Ok(content) = xtream::get_xtream_stream_info( - Arc::clone(&app_state.http_client.load()), + &app_state.http_client.load(), app_state, user, &input, @@ -1090,7 +1090,7 @@ async fn xtream_get_short_epg( // TODO serve epg from own db let input_source = InputSource::from(&*input).with_url(info_url); return match request::download_text_content( - Arc::clone(&app_state.http_client.load()), + &app_state.http_client.load(), None, &input_source, None, @@ -1243,7 +1243,7 @@ async fn xtream_get_catchup_response( let input_source = InputSource::from(&*input).with_url(info_url); let content = try_result_bad_request!( xtream::get_xtream_stream_info_content( - Arc::clone(&app_state.http_client.load()), + &app_state.http_client.load(), &input_source ) .await diff --git a/backend/src/api/main_api.rs b/backend/src/api/main_api.rs index 67e535ace..6d2948bbd 100644 --- a/backend/src/api/main_api.rs +++ b/backend/src/api/main_api.rs @@ -114,7 +114,7 @@ async fn create_shared_data( } fn exec_update_on_boot( - client: Arc, + client: &reqwest::Client, app_state: &Arc, targets: &Arc, ) { @@ -127,8 +127,9 @@ fn exec_update_on_boot( let app_state_clone = Arc::clone(&app_state.app_config); let targets_clone = Arc::clone(targets); let playlist_state = Arc::clone(&app_state.playlists); + let client = client.clone(); tokio::spawn(async move { - playlist::exec_processing(client, app_state_clone, targets_clone, None, Some(playlist_state)).await; + playlist::exec_processing(&client, app_state_clone, targets_clone, None, Some(playlist_state)).await; }); } } @@ -272,15 +273,17 @@ pub async fn start_server( exec_system_usage(&app_state); + let client = shared_data.http_client.load(); + exec_scheduler( - &Arc::clone(&shared_data.http_client.load()), + client.as_ref(), &app_state, &targets, &cancel_token_scheduler, ); exec_update_on_boot( - Arc::clone(&shared_data.http_client.load()), + client.as_ref(), &app_state, &targets, ); diff --git a/backend/src/api/model/streams/provider_stream_factory.rs b/backend/src/api/model/streams/provider_stream_factory.rs index cf30108c5..d001649c4 100644 --- a/backend/src/api/model/streams/provider_stream_factory.rs +++ b/backend/src/api/model/streams/provider_stream_factory.rs @@ -193,7 +193,7 @@ fn get_request_range_start_bytes(req_headers: &HashMap>) -> Opti // } fn prepare_client( - request_client: &Arc, + request_client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, ) -> (reqwest::RequestBuilder, bool) { let url = stream_options.get_url(); @@ -263,7 +263,7 @@ fn prepare_client( async fn provider_stream_request( app_state: &Arc, - request_client: &Arc, + request_client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, ) -> Result, StatusCode> { let (client, _partial_content) = prepare_client(request_client, stream_options); @@ -365,7 +365,7 @@ async fn handle_channel_unavailable_stream(app_state: &Arc, async fn get_provider_stream( app_state: &Arc, - client: &Arc, + client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, ) -> Result, StatusCode> { let url = stream_options.get_url(); @@ -419,7 +419,7 @@ async fn get_provider_stream( #[allow(clippy::too_many_lines)] pub async fn create_provider_stream( app_state: &Arc, - client: &Arc, + client: &reqwest::Client, stream_options: ProviderStreamFactoryOptions, ) -> Option { let client_stream_factory = |stream, reconnect_flag, range_cnt| { @@ -461,9 +461,9 @@ pub async fn create_provider_stream( let continue_streaming_signal = continue_client_signal.clone(); let stream_options_provider = stream_options.clone(); let app_state_clone = Arc::clone(app_state); - let client = Arc::clone(client); + let client = client.clone(); let unfold: BoxedProviderStream = stream::unfold((), move |()| { - let client = Arc::clone(&client); + let client = client.clone(); let stream_opts = stream_options_provider.clone(); let continue_streaming = continue_streaming_signal.clone(); let app_state_clone = Arc::clone(&app_state_clone); @@ -572,7 +572,7 @@ pub async fn create_provider_stream( // let input = None; // // let options = BufferStreamOptions::new(PlaylistItemType::Live, true, true, 0, false); -// let value = create_provider_stream(&cfg, Arc::clone(&client), &url, &req, input, options); +// let value = create_provider_stream(&cfg, &client, &url, &req, input, options); // let mut values = value.await; // 'outer: while let Some((ref mut stream, info)) = values.as_mut() { // if info.is_some() { diff --git a/backend/src/api/scheduler.rs b/backend/src/api/scheduler.rs index 0d8fa3768..7b511d092 100644 --- a/backend/src/api/scheduler.rs +++ b/backend/src/api/scheduler.rs @@ -26,7 +26,7 @@ pub fn datetime_to_instant(datetime: DateTime) -> Instant { Instant::now() + duration_until } -pub fn exec_scheduler(client: &Arc, app_state: &Arc, targets: &Arc, +pub fn exec_scheduler(client: &reqwest::Client, app_state: &Arc, targets: &Arc, cancel: &CancellationToken) { let cfg = &app_state.app_config; let config = cfg.config.load(); @@ -39,7 +39,7 @@ pub fn exec_scheduler(client: &Arc, app_state: &Arc, let expression = schedule.schedule.clone(); let exec_targets = get_process_targets(cfg, targets, schedule.targets.as_ref()); let app_state_clone = Arc::clone(app_state); - let http_client = Arc::clone(client); + let http_client = client.clone(); let cancel_token = cancel.clone(); tokio::spawn(async move { start_scheduler(http_client, expression.as_str(), app_state_clone, exec_targets, cancel_token).await; @@ -47,7 +47,7 @@ pub fn exec_scheduler(client: &Arc, app_state: &Arc, } } -async fn start_scheduler(client: Arc, expression: &str, app_state: Arc, +async fn start_scheduler(client: reqwest::Client, expression: &str, app_state: Arc, targets: Arc, cancel: CancellationToken) { match Schedule::from_str(expression) { Ok(schedule) => { @@ -55,12 +55,13 @@ async fn start_scheduler(client: Arc, expression: &str, app_sta loop { let mut upcoming = schedule.upcoming(offset).take(1); if let Some(datetime) = upcoming.next() { + let client = client.clone(); tokio::select! { () = tokio::time::sleep_until(tokio::time::Instant::from(datetime_to_instant(datetime))) => { let app_config = Arc::clone(&app_state.app_config); let event_manager = Arc::clone(&app_state.event_manager); let playlist_state = app_state.playlists.clone(); - exec_processing(Arc::clone(&client), app_config, Arc::clone(&targets), Some(event_manager), Some(playlist_state)).await; + exec_processing(&client, app_config, Arc::clone(&targets), Some(event_manager), Some(playlist_state)).await; } () = cancel.cancelled() => { break; diff --git a/backend/src/main.rs b/backend/src/main.rs index 706f29763..c186780aa 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -164,7 +164,7 @@ async fn start_in_cli_mode(cfg: Arc, targets: Arc) { error!("Failed to build client {err}"); reqwest::Client::new() }); - playlist::exec_processing(Arc::new(client), cfg, targets, None, None).await; + playlist::exec_processing(&client, cfg, targets, None, None).await; } async fn start_in_server_mode(cfg: Arc, targets: Arc) { diff --git a/backend/src/messaging.rs b/backend/src/messaging.rs index 8f7d6d610..54d6cd44b 100644 --- a/backend/src/messaging.rs +++ b/backend/src/messaging.rs @@ -5,13 +5,12 @@ use log::{debug, error}; use reqwest::header; use shared::model::MsgKind; use shared::utils::json_str_to_markdown; -use std::sync::Arc; fn is_enabled(kind: MsgKind, cfg: &MessagingConfig) -> bool { cfg.notify_on.contains(&kind) } -async fn send_http_post_request(client: &Arc, msg: &str, messaging: &MessagingConfig) { +async fn send_http_post_request(client: &reqwest::Client, msg: &str, messaging: &MessagingConfig) { if let Some(rest) = &messaging.rest { let data = msg.to_owned(); match client @@ -27,7 +26,7 @@ async fn send_http_post_request(client: &Arc, msg: &str, messag } } -async fn send_telegram_message(client: &Arc, msg: &str, messaging: &MessagingConfig, json: bool) { +async fn send_telegram_message(client: &reqwest::Client, msg: &str, messaging: &MessagingConfig, json: bool) { // TODO use proxy settings if let Some(telegram) = &messaging.telegram { let (message, options) = { @@ -49,7 +48,7 @@ async fn send_telegram_message(client: &Arc, msg: &str, messagi } } -async fn send_pushover_message(client: &Arc, msg: &str, messaging: &MessagingConfig) { +async fn send_pushover_message(client: &reqwest::Client, msg: &str, messaging: &MessagingConfig) { if let Some(pushover) = &messaging.pushover { let encoded_message: String = url::form_urlencoded::Serializer::new(String::new()) .append_pair("token", pushover.token.as_str()) @@ -75,7 +74,7 @@ async fn send_pushover_message(client: &Arc, msg: &str, messagi } } -async fn dispatch_send_message(client: &Arc, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str, json: bool) { +async fn dispatch_send_message(client: &reqwest::Client, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str, json: bool) { if let Some(messaging) = cfg { if is_enabled(kind, messaging) { tokio::join!( @@ -87,10 +86,10 @@ async fn dispatch_send_message(client: &Arc, kind: MsgKind, cfg } } -pub async fn send_message_json(client: &Arc, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) { +pub async fn send_message_json(client: &reqwest::Client, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) { dispatch_send_message(client, kind, cfg, msg, true).await; } -pub async fn send_message(client: &Arc, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) { +pub async fn send_message(client: &reqwest::Client, kind: MsgKind, cfg: Option<&MessagingConfig>, msg: &str) { dispatch_send_message(client, kind, cfg, msg, false).await; } diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index d4a5f131c..8954f41f2 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -197,8 +197,9 @@ pub async fn parse_xmltv_for_web_ui_from_file(path: &Path) -> Result, url: &str) -> Result { if let Ok(request_url) = Url::parse(url) { + let client = app_state.http_client.load(); match get_remote_content_as_stream( - Arc::clone(&app_state.http_client.load()), + client.as_ref(), &request_url, InputFetchMethod::GET, None, diff --git a/backend/src/processing/playlist_watch.rs b/backend/src/processing/playlist_watch.rs index d619f7f5c..a67033c6b 100644 --- a/backend/src/processing/playlist_watch.rs +++ b/backend/src/processing/playlist_watch.rs @@ -1,6 +1,5 @@ use std::collections::BTreeSet; use std::path::{Path}; -use std::sync::Arc; use log::{error, info}; use shared::model::{MsgKind, PlaylistGroup}; use crate::messaging::{send_message}; @@ -8,7 +7,7 @@ use crate::model::Config; use crate::utils; use crate::utils::{bincode_deserialize, bincode_serialize}; -pub async fn process_group_watch(client: &Arc, cfg: &Config, target_name: &str, pl: &PlaylistGroup) { +pub async fn process_group_watch(client: &reqwest::Client, cfg: &Config, target_name: &str, pl: &PlaylistGroup) { let mut new_tree = BTreeSet::new(); pl.channels.iter().for_each(|chan| { let header = &chan.header; @@ -60,7 +59,7 @@ struct WatchChanges { pub removed: Vec, } -async fn handle_watch_notification(client: &Arc, cfg: &Config, added: &BTreeSet, removed: &BTreeSet, target_name: &str, group_name: &str) { +async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added: &BTreeSet, removed: &BTreeSet, target_name: &str, group_name: &str) { let added = added.iter().map(std::string::ToString::to_string).collect::>(); let removed = removed.iter().map(std::string::ToString::to_string).collect::>(); if !added.is_empty() || !removed.is_empty() { diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index b168343f8..710bbf190 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -29,7 +29,6 @@ use crate::utils::StepMeasure; use deunicode::deunicode; use futures::StreamExt; use log::{debug, error, info, log_enabled, trace, warn, Level}; -use reqwest::Client; use shared::error::{get_errors_notify_message, notify_err, TuliproxError}; use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider}; use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster}; @@ -305,7 +304,7 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo (!user_targets.enabled && target.enabled) || (user_targets.enabled && user_targets.has_target(target.id)) } -async fn playlist_download_from_input(client: &Arc, config: &Arc, input: &Arc) -> (Vec, Vec) { +async fn playlist_download_from_input(client: &reqwest::Client, config: &Arc, input: &Arc) -> (Vec, Vec) { let working_dir = &config.working_dir; match input.input_type { InputType::M3u => m3u::get_m3u_playlist(client, config, input, working_dir).await, @@ -314,7 +313,7 @@ async fn playlist_download_from_input(client: &Arc, config: &Ar } } -async fn process_source(client: Arc, cfg: Arc, source_idx: usize, +async fn process_source(client: &reqwest::Client, cfg: Arc, source_idx: usize, user_targets: Arc, event_manager: Option>, playlist_state: Option<&Arc>, ) -> (Vec, Vec, Vec) { @@ -332,9 +331,9 @@ async fn process_source(client: Arc, cfg: Arc, sourc let working_dir = &config.working_dir; source_downloaded = true; let start_time = Instant::now(); - let (mut playlistgroups, mut error_list) = playlist_download_from_input(&client, &config, input).await; + let (mut playlistgroups, mut error_list) = playlist_download_from_input(client, &config, input).await; let (tvguide, mut tvguide_errors) = if error_list.is_empty() { - epg::get_xmltv(Arc::clone(&client), input, working_dir).await + epg::get_xmltv(client, input, working_dir).await } else { (None, vec![]) }; @@ -373,7 +372,7 @@ async fn process_source(client: Arc, cfg: Arc, sourc for target in &source.targets { let event_manager_clone = event_manager_clone.clone(); if is_target_enabled(target, &user_targets) { - match process_playlist_for_target(&cfg, Arc::clone(&client), &mut source_playlists, target, &mut input_stats, &mut errors, event_manager_clone, playlist_state).await { + match process_playlist_for_target(&cfg, client, &mut source_playlists, target, &mut input_stats, &mut errors, event_manager_clone, playlist_state).await { Ok(()) => { target_stats.push(TargetStats::success(&target.name)); } @@ -407,7 +406,7 @@ fn create_input_stat(group_count: usize, channel_count: usize, error_count: usiz } } -async fn process_sources(client: Arc, config: &Arc, user_targets: Arc, +async fn process_sources(client: &reqwest::Client, config: &Arc, user_targets: Arc, event_manager: Option>, playlist_state: Option<&Arc>, ) -> (Vec, Vec) { let mut async_tasks = JoinSet::new(); @@ -432,13 +431,13 @@ async fn process_sources(client: Arc, config: &Arc, let usr_trgts = user_targets.clone(); let event_manager = event_manager.clone(); if process_parallel { - let http_client = Arc::clone(&client); + let http_client = client.clone(); let playlist_state = playlist_state.cloned(); async_tasks.spawn(async move { // Hold the per-source lock for the full duration of this update. let current_update_lock = update_lock; let (input_stats, target_stats, mut res_errors) = - process_source(Arc::clone(&http_client), cfg, index, usr_trgts, event_manager, playlist_state.as_ref()).await; + process_source(&http_client, cfg, index, usr_trgts, event_manager, playlist_state.as_ref()).await; shared_errors.lock().await.append(&mut res_errors); if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) { shared_stats.lock().await.push(process_stats); @@ -447,7 +446,7 @@ async fn process_sources(client: Arc, config: &Arc, }); } else { let (input_stats, target_stats, mut res_errors) = - process_source(Arc::clone(&client), cfg, index, usr_trgts, event_manager, playlist_state).await; + process_source(client, cfg, index, usr_trgts, event_manager, playlist_state).await; shared_errors.lock().await.append(&mut res_errors); if let Some(process_stats) = SourceStats::try_new(input_stats, target_stats) { shared_stats.lock().await.push(process_stats); @@ -531,7 +530,7 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { #[allow(clippy::too_many_arguments)] async fn process_playlist_for_target(app_config: &AppConfig, - client: Arc, + client: &reqwest::Client, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, stats: &mut HashMap, @@ -558,8 +557,8 @@ async fn process_playlist_for_target(app_config: &AppConfig, let mut step = StepMeasure::new(&target.name, broadcast_step); for provider_fpl in playlists.iter_mut() { let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); - playlist_resolve_series(app_config, Arc::clone(&client), target, errors, &pipe, provider_fpl, &mut processed_fpl).await; - playlist_resolve_vod(app_config, Arc::clone(&client), target, errors, &mut processed_fpl).await; + playlist_resolve_series(app_config, client, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; + playlist_resolve_vod(app_config, client, target, errors, &mut processed_fpl).await; // stats let input_stats = stats.get_mut(&processed_fpl.input.name); if let Some(stat) = input_stats { @@ -580,7 +579,7 @@ async fn process_playlist_for_target(app_config: &AppConfig, Ok(()) } else { // Process Trakt categories - if trakt_playlist(&client, target, errors, &mut new_playlist).await { + if trakt_playlist(client, target, errors, &mut new_playlist).await { step.tick("trakt categories"); } @@ -596,7 +595,7 @@ async fn process_playlist_for_target(app_config: &AppConfig, step.tick("assigning channel counter"); let config = app_config.config.load(); - if process_watch(&config, &client, target, &flat_new_playlist).await { + if process_watch(&config, client, target, &flat_new_playlist).await { step.tick("group watches"); } let result = persist_playlist(app_config, &mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, playlist_state).await; @@ -605,8 +604,8 @@ async fn process_playlist_for_target(app_config: &AppConfig, } } -async fn trakt_playlist(client: &Arc, target: &ConfigTarget, errors: &mut Vec, playlist: &mut Vec) -> bool { - match process_trakt_categories_for_target(Arc::clone(client), playlist, target).await { +async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, playlist: &mut Vec) -> bool { + match process_trakt_categories_for_target(client, playlist, target).await { Ok(Some(trakt_categories)) => { if !trakt_categories.is_empty() { info!("Adding {} Trakt categories to playlist", trakt_categories.len()); @@ -637,7 +636,7 @@ async fn process_epg(processed_fetched_playlists: &mut Vec>) (new_epg, new_playlist) } -async fn process_watch(cfg: &Config, client: &Arc, target: &ConfigTarget, new_playlist: &[PlaylistGroup]) -> bool { +async fn process_watch(cfg: &Config, client: &reqwest::Client, target: &ConfigTarget, new_playlist: &[PlaylistGroup]) -> bool { if let Some(watches) = &target.watch { if default_as_default().eq_ignore_ascii_case(&target.name) { error!("can't watch a target with no unique name"); @@ -657,10 +656,10 @@ async fn process_watch(cfg: &Config, client: &Arc, target: &Con } } -pub async fn exec_processing(client: Arc, app_config: Arc, targets: Arc, event_manager: Option>, playlist_state: Option>) { +pub async fn exec_processing(client: &reqwest::Client, app_config: Arc, targets: Arc, event_manager: Option>, playlist_state: Option>) { let start_time = Instant::now(); let event_manager_clone = event_manager.clone(); - let (stats, errors) = process_sources(Arc::clone(&client), &app_config, targets.clone(), event_manager_clone, playlist_state.as_ref()).await; + let (stats, errors) = process_sources(client, &app_config, targets.clone(), event_manager_clone, playlist_state.as_ref()).await; // log errors for err in &errors { error!("{}", err.message); @@ -677,7 +676,7 @@ pub async fn exec_processing(client: Arc, app_config: Arc error!("Failed to serialize playlist stats {err}"), } @@ -692,7 +691,7 @@ pub async fn exec_processing(client: Arc, app_config: Arc Option<&str> { @@ -243,8 +242,8 @@ pub struct TraktCategoriesProcessor { } impl TraktCategoriesProcessor { - pub fn new(http_client: Arc, trakt_config: &TraktConfig) -> Self { - let client = TraktClient::new(http_client, trakt_config.api.clone()); + pub fn new(http_client: &reqwest::Client, trakt_config: &TraktConfig) -> Self { + let client = TraktClient::new(http_client.clone(), trakt_config.api.clone()); Self { client } } @@ -292,7 +291,7 @@ impl TraktCategoriesProcessor { } } pub async fn process_trakt_categories_for_target( - http_client: Arc, + http_client: &reqwest::Client, playlist: &[PlaylistGroup], target: &ConfigTarget, ) -> Result>, Vec> { diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index f7591d133..9d18c423d 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs::File; use std::path::PathBuf; -use std::sync::Arc; use crate::repository::bplustree::BPlusTree; use crate::repository::storage_const; use crate::repository::xtream_repository::xtream_get_record_file_path; @@ -17,7 +16,7 @@ use crate::utils; use crate::utils::xtream; use serde_json::{from_str, to_string, Value}; -pub(in crate::processing) async fn playlist_resolve_download_playlist_item(client: Arc, pli: &PlaylistItem, input: &ConfigInput, errors: &mut Vec, resolve_delay: u16, cluster: XtreamCluster) -> Option { +pub(in crate::processing) async fn playlist_resolve_download_playlist_item(client: &reqwest::Client, pli: &PlaylistItem, input: &ConfigInput, errors: &mut Vec, resolve_delay: u16, cluster: XtreamCluster) -> Option { let mut result = None; let provider_id = pli.get_provider_id()?; if let Some(info_url) = xtream::get_xtream_player_api_info_url(input, cluster, provider_id) { diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 9389cb6e8..6a788132b 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -14,7 +14,6 @@ use crate::processing::processor::{handle_error, handle_error_and_return, create use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{BufWriter, Write}; -use std::sync::Arc; use std::time::Instant; use log::{error, info, log_enabled, warn, Level}; use crate::model::{XtreamSeriesEpisode, XtreamSeriesInfoEpisode}; @@ -50,7 +49,7 @@ fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &Ha should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_SERIES_INFO_LAST_MODIFIED) } -async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc, errors: &mut Vec, +async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, fpl).await; let mut fetched_in_run: HashSet = HashSet::new(); @@ -83,7 +82,7 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc, + client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, pipe: &ProcessingPipe, diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 865c1b506..8880fb509 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -8,11 +8,9 @@ use crate::repository::xtream_repository::{write_vod_info_to_wal_file, xtream_up use shared::error::{notify_err}; use crate::processing::processor::{handle_error, handle_error_and_return, create_resolve_options_function_for_xtream_target}; use shared::utils::{get_u32_from_serde_value, get_u64_from_serde_value, get_string_from_serde_value}; -use crate::repository::xtream_repository::xtream_get_input_info; use serde_json::{from_str, Map, Value}; use std::collections::{HashMap, HashSet}; use std::io::{Write}; -use std::sync::Arc; use std::time::Instant; use log::{info, log_enabled, Level}; use crate::utils; @@ -65,7 +63,9 @@ fn should_update_vod_info(pli: &mut PlaylistItem, processed_provider_ids: &HashM should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_VOD_INFO_ADDED) } -pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc, target: &ConfigTarget, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>) { +const FLUSH_INTERVAL: usize = 50; + +pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>) { let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl); if !resolve_movies { return; } @@ -76,7 +76,7 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc = read_processed_vod_info_ids(app_config, errors, fpl).await; let mut fetched_in_run: HashSet = HashSet::new(); let mut content_writer = utils::file_writer(&wal_content_file); let mut record_writer = utils::file_writer(&wal_record_file); @@ -87,78 +87,79 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc>(normalized_str).ok().and_then(|info_doc| { + info_doc.get("info").cloned().map(|info_content| { + let mut wrapped_info = Map::new(); + wrapped_info.insert("info".to_string(), info_content); + Value::Object(wrapped_info) + }) + }); + } + } + } + if log_enabled!(Level::Info) { + processed_vod_info_count += 1; + if last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); + last_log_time = Instant::now(); } } } - if log_enabled!(Level::Info) { - processed_vod_info_count += 1; - let elapsed = start_time.elapsed().as_secs(); - if elapsed > 0 && ((processed_vod_info_count - last_processed_vod_info_count) > 50) && elapsed.is_multiple_of(30) { - info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); - last_processed_vod_info_count = processed_vod_info_count; - } - } - } - if last_processed_vod_info_count != processed_vod_info_count { - info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); } + info!("resolved {processed_vod_info_count}/{vod_info_count} vod info"); if content_updated { // TODO better approach for transactional updates is multiplexed WAL file. - + // final flush & sync with proper error handling handle_error!(content_writer.flush(), |err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}")))); handle_error!(record_writer.flush(), |err| errors.push(notify_err!(format!("Failed to resolve vod tmdb, could not write to wal file {err}")))); handle_error!(content_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync vod info to wal file {err}")))); handle_error!(record_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync vod info record to wal file {err}")))); + // drop writers and files to release handles drop(content_writer); drop(record_writer); drop(wal_content_file); drop(wal_record_file); + handle_error!(xtream_update_input_info_file(app_config, fpl.input, &wal_content_path, XtreamCluster::Video).await, |err| errors.push(err)); handle_error!(xtream_update_input_vod_record_from_wal_file(app_config, fpl.input, &wal_record_path).await, |err| errors.push(err)); } - - // Update in-memory playlist items with the newly fetched vod info. - // This makes the data available for subsequent processing steps like STRM export. - let vod_info_iter = fpl.playlistgroups.iter_mut() - .flat_map(|plg| &mut plg.channels) - .filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video); - - for pli in vod_info_iter { - if let Some(provider_id) = pli.header.get_provider_id() { - if let Some(content) = xtream_get_input_info(app_config, fpl.input, provider_id, XtreamCluster::Video).await { - // Add the "info" section to the playlist item additional properties. - pli.header.additional_properties = from_str::>(&content).ok().and_then(|info_doc| { - info_doc.get("info").cloned().map(|info_content| { - let mut wrapped_info = Map::new(); - wrapped_info.insert("info".to_string(), info_content); - Value::Object(wrapped_info) - }) - }); - } - } - } } diff --git a/backend/src/utils/network/epg.rs b/backend/src/utils/network/epg.rs index 2dbe95ee9..1e51b8a90 100644 --- a/backend/src/utils/network/epg.rs +++ b/backend/src/utils/network/epg.rs @@ -5,7 +5,6 @@ use crate::utils::{add_prefix_to_filename, cleanup_unlisted_files_with_suffix, p use crate::utils::request; use log::debug; use std::path::PathBuf; -use std::sync::Arc; use shared::utils::{sanitize_sensitive_info, short_hash}; fn get_input_raw_epg_file_path(url: &str, input: &ConfigInput, working_dir: &str) -> Option { @@ -14,13 +13,13 @@ fn get_input_raw_epg_file_path(url: &str, input: &ConfigInput, working_dir: &str .map(|path| add_prefix_to_filename(&path, format!("{file_prefix}_epg_").as_str(), Some("xml"))) } -async fn download_epg_file(url: &str, client: &Arc, input: &ConfigInput, working_dir: &str) -> Result { +async fn download_epg_file(url: &str, client: &reqwest::Client, input: &ConfigInput, working_dir: &str) -> Result { debug!("Getting epg file path for url: {}", sanitize_sensitive_info(url)); let persist_file_path = get_input_raw_epg_file_path(url, input, working_dir); - request::get_input_epg_content_as_file(Arc::clone(client), input, working_dir, url, persist_file_path).await + request::get_input_epg_content_as_file(client, input, working_dir, url, persist_file_path).await } -pub async fn get_xmltv(client: Arc, input: &ConfigInput, working_dir: &str) -> (Option, Vec) { +pub async fn get_xmltv(client: &reqwest::Client, input: &ConfigInput, working_dir: &str) -> (Option, Vec) { match &input.epg { None => (None, vec![]), Some(epg_config) => { @@ -29,7 +28,7 @@ pub async fn get_xmltv(client: Arc, input: &ConfigInput, workin let mut stored_file_paths = vec![]; for epg_source in &epg_config.sources { - match download_epg_file(&epg_source.url, &client, input, working_dir).await { + match download_epg_file(&epg_source.url, client, input, working_dir).await { Ok(file_path) => { stored_file_paths.push(file_path.clone()); file_paths.push(PersistedEpgSource {file_path, priority: epg_source.priority, logo_override: epg_source.logo_override}); diff --git a/backend/src/utils/network/ip_checker.rs b/backend/src/utils/network/ip_checker.rs index 230f5a235..1ee5dcdf4 100644 --- a/backend/src/utils/network/ip_checker.rs +++ b/backend/src/utils/network/ip_checker.rs @@ -1,11 +1,9 @@ use shared::error::{TuliproxError, TuliproxErrorKind}; use crate::model::IpCheckConfig; use regex::Regex; -use reqwest::Client; -use std::sync::Arc; use shared::utils::sanitize_sensitive_info; -async fn fetch_ip(client: &Arc, url: &str, regex: Option<&Regex>) -> Result { +async fn fetch_ip(client: &reqwest::Client, url: &str, regex: Option<&Regex>) -> Result { let response = client.get(url).send().await .map_err(|e| TuliproxError::new(TuliproxErrorKind::Info, format!("Failed to request {}: {e}", sanitize_sensitive_info(url))))?; @@ -28,7 +26,7 @@ async fn fetch_ip(client: &Arc, url: &str, regex: Option<&Regex>) -> Res } /// Fetch both IPs from a shared URL (if both regex patterns are available) -async fn fetch_combined_ips(client: &Arc, config: &IpCheckConfig, url: &str) -> (Option, Option) { +async fn fetch_combined_ips(client: &reqwest::Client, config: &IpCheckConfig, url: &str) -> (Option, Option) { let response = client.get(url).send().await.ok(); let text = match response { Some(r) => r.text().await.ok(), @@ -55,7 +53,7 @@ async fn fetch_combined_ips(client: &Arc, config: &IpCheckConfig, url: & } /// Fetch both IPv4 and IPv6 addresses, using separate or combined URL(s) -pub async fn get_ips(client: &Arc, config: &IpCheckConfig) -> Result<(Option, Option), TuliproxError> { +pub async fn get_ips(client: &reqwest::Client, config: &IpCheckConfig) -> Result<(Option, Option), TuliproxError> { match (&config.url_ipv4, &config.url_ipv6, &config.url) { // Both dedicated URLs provided (Some(url_v4), Some(url_v6), _) => { diff --git a/backend/src/utils/network/m3u.rs b/backend/src/utils/network/m3u.rs index a29e0351a..d640d177e 100644 --- a/backend/src/utils/network/m3u.rs +++ b/backend/src/utils/network/m3u.rs @@ -6,7 +6,7 @@ use crate::processing::parser::m3u; use crate::utils::prepare_file_path; use crate::utils::request; -pub async fn get_m3u_playlist(client: &Arc, cfg: &Arc, input: &Arc, working_dir: &str) -> (Vec, Vec) { +pub async fn get_m3u_playlist(client: &reqwest::Client, cfg: &Arc, input: &Arc, working_dir: &str) -> (Vec, Vec) { let input_source: InputSource = { match input.staged.as_ref() { None => input.as_ref().into(), diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 8c907fd28..4b15cde7b 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -2,7 +2,6 @@ use std::collections::{HashMap, HashSet}; use std::io::{Error, ErrorKind}; use std::path::{Path, PathBuf}; use std::pin::Pin; -use std::sync::Arc; use std::time::{Duration, Instant}; use futures::{StreamExt, TryStreamExt}; use log::{debug, error, log_enabled, trace, Level}; @@ -53,7 +52,7 @@ pub fn classify_content_type(headers: &[(String, String)]) -> MimeCategory { }) } -pub async fn get_input_epg_content_as_file(client: Arc, input: &ConfigInput, working_dir: &str, url_str: &str, persist_filepath: Option) -> Result { +pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &ConfigInput, working_dir: &str, url_str: &str, persist_filepath: Option) -> Result { debug_if_enabled!("getting input epg content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(url_str)); if url_str.parse::().is_ok() { match download_epg_content_as_file(client, input, url_str, working_dir, persist_filepath).await { @@ -96,11 +95,11 @@ pub async fn get_input_epg_content_as_file(client: Arc, input: } -pub async fn get_input_text_content(client: &Arc, input: &InputSource, working_dir: &str, persist_filepath: Option) -> Result { +pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option) -> Result { debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url)); if input.url.parse::().is_ok() { - match download_text_content(Arc::clone(client), None, input, None, persist_filepath).await { + match download_text_content(client, None, input, None, persist_filepath).await { Ok((content, _response_url)) => Ok(content), Err(e) => { error!("Failed to download input '{}': {}", &input.name, sanitize_sensitive_info(e.to_string().as_str())); @@ -140,7 +139,7 @@ pub async fn get_input_text_content(client: &Arc, input: &Input } pub fn get_client_request - (client: &Arc, + (client: &reqwest::Client, method: InputFetchMethod, headers: Option<&HashMap>, url: &Url, @@ -242,9 +241,9 @@ pub async fn get_local_file_content(file_path: &Path) -> Result, input: &ConfigInput, url: &Url, file_path: &Path) -> Result { +async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInput, url: &Url, file_path: &Path) -> Result { let start_time = Instant::now(); - let request = get_client_request(&client, input.method, Some(&input.headers), url, None, None); + let request = get_client_request(client, input.method, Some(&input.headers), url, None, None); match request.send().await { Ok(response) => { if response.status().is_success() { @@ -279,12 +278,12 @@ type DynReader = Pin>; #[allow(clippy::implicit_hasher)] pub async fn get_remote_content_as_stream( - client: Arc, + client: &reqwest::Client, url: &Url, method: InputFetchMethod, headers: Option<&HashMap> ) -> Result<(DynReader, String), Error> { - let request = get_client_request(&client, method, headers, url, None, None); + let request = get_client_request(client, method, headers, url, None, None); let response = request.send().await.map_err(std::io::Error::other)?; if !response.status().is_success() { @@ -320,7 +319,7 @@ pub async fn get_remote_content_as_stream( Ok((reader, response_url)) } -async fn get_remote_content(client: Arc, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> { +async fn get_remote_content(client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> { let start_time = Instant::now(); let custom_headers = headers.map(|h| { @@ -328,14 +327,14 @@ async fn get_remote_content(client: Arc, input: &InputSource, h let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers); let headers: HashMap = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect(); - let (mut stream, response_url) = get_remote_content_as_stream(client.clone(), url, input.method, Some(&headers)).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?; + let (mut stream, response_url) = get_remote_content_as_stream(client, url, input.method, Some(&headers)).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?; let mut content = String::new(); stream.read_to_string(&mut content).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?; debug_if_enabled!("Request took: {} {}", format_elapsed_time(start_time.elapsed().as_secs()), sanitize_sensitive_info(url.as_str())); Ok((content, response_url)) } -async fn download_epg_content_as_file(client: Arc, input: &ConfigInput, url_str: &str, working_dir: &str, persist_filepath: Option) -> Result { +async fn download_epg_content_as_file(client: &reqwest::Client, input: &ConfigInput, url_str: &str, working_dir: &str, persist_filepath: Option) -> Result { if let Ok(url) = url_str.parse::() { if url.scheme() == "file" { url.to_file_path().map_or_else(|()| Err(Error::new(ErrorKind::Unsupported, format!("Unknown file {}", sanitize_sensitive_info(url_str)))), |file_path| if file_path.exists() { @@ -361,7 +360,7 @@ async fn download_epg_content_as_file(client: Arc, input: &Conf } pub async fn download_text_content( - client: Arc, + client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, headers: Option<&HeaderMap>, @@ -396,7 +395,7 @@ pub async fn download_text_content( } } -async fn download_json_content(client: Arc, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { +async fn download_json_content(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url)); match download_text_content(client, disabled_headers, input, None, persist_filepath).await { Ok((content, _response_url)) => { @@ -409,7 +408,7 @@ async fn download_json_content(client: Arc, disabled_headers: O } } -pub async fn get_input_json_content(client: Arc, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { +pub async fn get_input_json_content(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { match download_json_content(client, disabled_headers, input, persist_filepath).await { Ok(content) => Ok(content), Err(e) => create_tuliprox_error_result!(TuliproxErrorKind::Notify, "cant download input {}, url: {} => {}", input.name, sanitize_sensitive_info(&input.url), sanitize_sensitive_info(e.to_string().as_str())) diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index 2e126b40e..005345bd0 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -46,7 +46,7 @@ pub fn get_xtream_player_api_info_url(input: &ConfigInput, cluster: XtreamCluste } -pub async fn get_xtream_stream_info_content(client: Arc, input: &InputSource) -> Result { +pub async fn get_xtream_stream_info_content(client: &reqwest::Client, input: &InputSource) -> Result { match request::download_text_content(client, None, input, None, None).await { Ok((content, _response_url)) => Ok(content), Err(err) => Err(err) @@ -54,7 +54,7 @@ pub async fn get_xtream_stream_info_content(client: Arc, input: } #[allow(clippy::too_many_arguments)] -pub async fn get_xtream_stream_info

(client: Arc, +pub async fn get_xtream_stream_info

(client: &reqwest::Client, app_state: &Arc, user: &ProxyUserCredentials, input: &ConfigInput, @@ -135,12 +135,12 @@ const ACTIONS: [(XtreamCluster, &str, &str); 3] = [ (XtreamCluster::Video, crate::model::XC_ACTION_GET_VOD_CATEGORIES, crate::model::XC_ACTION_GET_VOD_STREAMS), (XtreamCluster::Series, crate::model::XC_ACTION_GET_SERIES_CATEGORIES, crate::model::XC_ACTION_GET_SERIES)]; -async fn xtream_login(cfg: &Config, client: &Arc, input: &InputSource, username: &str) -> Result, TuliproxError> { - let content = if let Ok(content) = request::get_input_json_content(Arc::clone(client), None, input, None).await { +async fn xtream_login(cfg: &Config, client: &reqwest::Client, input: &InputSource, username: &str) -> Result, TuliproxError> { + let content = if let Ok(content) = request::get_input_json_content(client, None, input, None).await { content } else { let input_source_account_info = input.with_url(format!("{}&action={}", &input.url, crate::model::XC_ACTION_GET_ACCOUNT_INFO)); - match request::get_input_json_content(Arc::clone(client), None, &input_source_account_info, None).await { + match request::get_input_json_content(client, None, &input_source_account_info, None).await { Ok(content) => content, Err(err) => { warn!("Failed to login xtream account {username} {err}"); @@ -182,7 +182,7 @@ async fn xtream_login(cfg: &Config, client: &Arc, input: &Input } } -pub async fn notify_account_expire(exp_date: Option, cfg: &Config, client: &Arc, username: &str, input_name: &str) { +pub async fn notify_account_expire(exp_date: Option, cfg: &Config, client: &reqwest::Client, username: &str, input_name: &str) { if let Some(expiration_timestamp) = exp_date { let now_secs = Utc::now().timestamp(); // UTC-Time if expiration_timestamp > now_secs { @@ -202,7 +202,7 @@ pub async fn notify_account_expire(exp_date: Option, cfg: &Config, client: } } -pub async fn get_xtream_playlist(cfg: &Arc, client: &Arc, input: &Arc, working_dir: &str) -> (Vec, Vec) { +pub async fn get_xtream_playlist(cfg: &Arc, client: &reqwest::Client, input: &Arc, working_dir: &str) -> (Vec, Vec) { let input_source: InputSource = { match input.staged.as_ref() { None => input.as_ref().into(), @@ -235,8 +235,8 @@ pub async fn get_xtream_playlist(cfg: &Arc, client: &Arc { match xtream::parse_xtream(input, @@ -265,7 +265,7 @@ pub async fn get_xtream_playlist(cfg: &Arc, client: &Arc, client: &Arc, input: &Arc) { +async fn check_alias_user_state(cfg: &Arc, client: &reqwest::Client, input: &Arc) { if let Some(aliases) = input.aliases.as_ref() { for alias in aliases { if is_input_expired(alias.exp_date) { diff --git a/backend/src/utils/telegram.rs b/backend/src/utils/telegram.rs index aaae5c008..4755775fb 100644 --- a/backend/src/utils/telegram.rs +++ b/backend/src/utils/telegram.rs @@ -1,4 +1,3 @@ -use std::sync::Arc; use log::{debug, error}; use url::Url; @@ -63,7 +62,7 @@ pub fn telegram_create_instance(bot_token: &str, chat_id: &str) -> BotInstance { } pub async fn telegram_send_message( - client: &Arc, + client: &reqwest::Client, instance: &BotInstance, msg: &str, options: Option<&SendMessageOption>, diff --git a/backend/src/utils/trakt/client.rs b/backend/src/utils/trakt/client.rs index 43adf51e6..ae2c473ee 100644 --- a/backend/src/utils/trakt/client.rs +++ b/backend/src/utils/trakt/client.rs @@ -1,21 +1,20 @@ use crate::model::{TraktApiConfig, TraktListConfig, TraktListItem}; use shared::error::TuliproxError; use reqwest::header::{HeaderMap, HeaderValue}; -use std::sync::Arc; use log::{debug, info}; use shared::error::{info_err}; use shared::utils::trim_last_slash; use super::errors::{handle_trakt_api_error}; pub struct TraktClient { - client: Arc, + client: reqwest::Client, api_config: TraktApiConfig, // Pre-computed headers to avoid recreating them each time headers: HeaderMap, } impl TraktClient { - pub fn new(client: Arc, api_config: TraktApiConfig) -> Self { + pub fn new(client: reqwest::Client, api_config: TraktApiConfig) -> Self { let headers = Self::create_headers(&api_config); Self { client, From ab8e6c0d53db086a362a4005af540bc43eaaa322 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 14:32:04 +0100 Subject: [PATCH 02/11] write operations optimized for less memory usage --- backend/src/api/endpoints/download_api.rs | 16 +- .../api/model/streams/persist_pipe_stream.rs | 11 + backend/src/model/xmltv.rs | 14 +- .../src/processing/processor/xtream_series.rs | 86 ++++--- backend/src/repository/bplustree.rs | 241 ++++++++++-------- backend/src/repository/epg_repository.rs | 2 - backend/src/utils/file/file_utils.rs | 25 +- 7 files changed, 241 insertions(+), 154 deletions(-) diff --git a/backend/src/api/endpoints/download_api.rs b/backend/src/api/endpoints/download_api.rs index 3ec401d1f..ff69feec6 100644 --- a/backend/src/api/endpoints/download_api.rs +++ b/backend/src/api/endpoints/download_api.rs @@ -1,7 +1,7 @@ use crate::api::model::AppState; use crate::api::model::{DownloadQueue, FileDownload, FileDownloadRequest}; use crate::model::{AppConfig, VideoDownloadConfig}; -use crate::utils::request; +use crate::utils::{async_file_writer, request}; use tokio::sync::RwLock; use futures::stream::TryStreamExt; use log::info; @@ -25,15 +25,23 @@ async fn download_file(active: Arc>>, client: &reqwe if let Some(file_path_str) = file_download.file_path.to_str() { info!("Downloading {file_path_str}"); match File::create(&file_download.file_path).await { - Ok(mut file) => { + Ok(file) => { + let mut buf_writer = async_file_writer(file); let mut downloaded: u64 = 0; let mut stream = response.bytes_stream().map_err(to_io_error); + let mut write_counter = 0; loop { match stream.try_next().await { Ok(item) => { if let Some(chunk) = item { - match file.write_all(&chunk).await { + match buf_writer.write_all(&chunk).await { Ok(()) => { + write_counter += 1; + if write_counter > 50 { + buf_writer.flush().await.map_err(|err| err.to_string())?; + write_counter = 0; + } + downloaded += chunk.len() as u64; if let Some(lock) = active.write().await.as_mut() { lock.size = downloaded; @@ -47,6 +55,8 @@ async fn download_file(active: Arc>>, client: &reqwe if let Some(lock) = active.write().await.as_mut() { lock.size = downloaded; } + buf_writer.flush().await.map_err(|err| err.to_string())?; + buf_writer.shutdown().await.map_err(|err| err.to_string())?; return Ok(()); } } diff --git a/backend/src/api/model/streams/persist_pipe_stream.rs b/backend/src/api/model/streams/persist_pipe_stream.rs index 10f48880e..984ec1001 100644 --- a/backend/src/api/model/streams/persist_pipe_stream.rs +++ b/backend/src/api/model/streams/persist_pipe_stream.rs @@ -7,6 +7,8 @@ use tokio::io::AsyncWriteExt; use tokio_stream::{StreamExt}; use tokio_stream::wrappers::ReceiverStream; +const FLUSH_INTERVAL: usize = 50; + pub fn tee_stream( mut stream: S, mut writer: W, @@ -23,6 +25,7 @@ where S: tokio_stream::Stream> + Send + Unpin let mut total_size = 0usize; let mut writer_active = true; let mut write_err: Option = None; + let mut write_counter = 0usize; while let Some(chunk) = stream.next().await { match chunk { @@ -32,6 +35,14 @@ where S: tokio_stream::Stream> + Send + Unpin if let Err(e) = writer.write_all(&bytes).await { writer_active = false; write_err = Some(StreamError::StdIo(e.to_string())); + } else { + write_counter += 1; + if write_counter > FLUSH_INTERVAL { + write_counter = 0; + if let Err(err) = writer.flush().await { + write_err = Some(StreamError::StdIo(format!("Failed periodic flush of tee_stream writer {err}"))); + } + } } } diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index 8954f41f2..7896246b3 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use futures::TryFutureExt; -use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use url::Url; use shared::utils::sanitize_sensitive_info; use crate::api::model::AppState; @@ -93,6 +93,8 @@ impl Epg { .map(|c| (c, false)) .collect(); + let mut write_counter = 0usize; + while let Some((tag, ended)) = stack.pop() { if ended { // End-Event @@ -122,10 +124,20 @@ impl Epg { } } } + write_counter += 1; + if write_counter > 50 { + writer.get_mut().flush().await?; // flush underlying writer + write_counter = 0; + } } // write tv-end writer.write_event_async(Event::End(BytesEnd::new("tv"))).await?; + + let inner = writer.get_mut(); + inner.flush().await?; + inner.shutdown().await?; + Ok(()) } } diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 6a788132b..158388dd8 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -49,9 +49,11 @@ fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &Ha should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_SERIES_INFO_LAST_MODIFIED) } +const FLUSH_INTERVAL: usize = 50; + async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { - let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, fpl).await; + let mut processed_info_ids: HashMap = read_processed_series_info_ids(cfg, errors, fpl).await; let mut fetched_in_run: HashSet = HashSet::new(); // we cant write to the indexed-document directly because of the write lock and time-consuming operation. // All readers would be waiting for the lock and the app would be unresponsive. @@ -63,45 +65,57 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, let mut record_writer = utils::file_writer(&wal_record_file); let mut content_updated = false; - // TODO merge both filters to one let series_info_count = fpl.playlistgroups.iter() .filter(|&plg| plg.xtream_cluster == XtreamCluster::Series) .flat_map(|plg| &plg.channels) .filter(|&pli| pli.header.item_type == PlaylistItemType::SeriesInfo).count(); - let series_info_iter = fpl.playlistgroups.iter_mut() - .filter(|plg| plg.xtream_cluster == XtreamCluster::Series) - .flat_map(|plg| &mut plg.channels) - .filter(|pli| pli.header.item_type == PlaylistItemType::SeriesInfo); - info!("Found {series_info_count} series info to resolve"); - let start_time = Instant::now(); + let mut last_log_time = Instant::now(); let mut processed_series_info_count = 0; - let mut last_processed_series_info_count = 0; - for pli in series_info_iter { - let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids); - if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) { - if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await { - let normalized_content = normalize_json_content(content); - handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, &normalized_content, &mut content_writer, &mut record_writer), - |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}")))); - processed_info_ids.insert(provider_id, ts); - content_updated = true; - } + let mut write_counter = 0usize; + + for plg in &mut fpl.playlistgroups { + if plg.xtream_cluster != XtreamCluster::Series { + continue; } - if log_enabled!(Level::Info) { - processed_series_info_count += 1; - let elapsed = start_time.elapsed().as_secs(); - if elapsed > 0 && ((processed_series_info_count - last_processed_series_info_count) > 50) && elapsed.is_multiple_of(30) { - info!("resolved {processed_series_info_count}/{series_info_count} series info"); - last_processed_series_info_count = processed_series_info_count; + for pli in &mut plg.channels { + if pli.header.item_type != PlaylistItemType::SeriesInfo { + continue; + } + let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids); + if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) { + if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await { + let normalized_content = normalize_json_content(content); + let normalized_str = normalized_content.as_str(); + handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, normalized_str, &mut content_writer, &mut record_writer), + |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}")))); + processed_info_ids.insert(provider_id, ts); + content_updated = true; + write_counter += 1; + + // periodic flush to bound BufWriter memory + if write_counter.is_multiple_of(FLUSH_INTERVAL) { + if let Err(err) = content_writer.flush() { + errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); + } + if let Err(err) = record_writer.flush() { + errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}"))); + } + } + } + } + if log_enabled!(Level::Info) { + processed_series_info_count += 1; + if last_log_time.elapsed().as_secs() >= 30 { + info!("resolved {processed_series_info_count}/{series_info_count} series info"); + last_log_time = Instant::now(); + } } } } - if last_processed_series_info_count != processed_series_info_count { - info!("resolved {processed_series_info_count}/{series_info_count} series info"); - } + info!("resolved {processed_series_info_count}/{series_info_count} series info"); // content_wal contains the provider_id and series_info with episode listing // record_wal contains provider_id and timestamp if content_updated { @@ -112,8 +126,8 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, handle_error!(content_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}")))); handle_error!(record_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info record to wal file {err}")))); drop(content_writer); - drop(wal_content_file); drop(record_writer); + drop(wal_content_file); drop(wal_record_file); handle_error!(xtream_update_input_info_file(cfg, fpl.input, &wal_content_path, XtreamCluster::Series).await, |err| errors.push(err)); @@ -142,6 +156,8 @@ async fn process_series_info( return result; }; + let mut write_counter = 0usize; + let _file_lock = app_config.file_locks.read_lock(&info_path).await; // Contains the Series Info with episode listing @@ -184,6 +200,13 @@ async fn process_series_info( handle_error!(write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, episode), |err| errors.push(info_err!(format!("Failed to write to series episode wal file: {err}")))); } + write_counter +=1; + // periodic flush to bound BufWriter memory + if write_counter.is_multiple_of(FLUSH_INTERVAL) { + if let Err(err) = wal_writer.flush() { + errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); + } + } group_series.extend(series.into_iter().map(|(_, pli)| pli)); } Ok(None) => {} @@ -205,8 +228,9 @@ async fn process_series_info( } } - handle_error!(wal_writer.flush(), - |err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}")))); + handle_error!(wal_writer.flush(), |err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}")))); + handle_error!(wal_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}")))); + drop(wal_writer); drop(wal_file); handle_error!(xtream_update_input_series_episodes_record_from_wal_file(app_config, input, &wal_path).await, diff --git a/backend/src/repository/bplustree.rs b/backend/src/repository/bplustree.rs index c5576d1bf..5b15f4eef 100644 --- a/backend/src/repository/bplustree.rs +++ b/backend/src/repository/bplustree.rs @@ -1,7 +1,7 @@ use crate::utils; use log::error; use ruzstd::decoding::StreamingDecoder; -use ruzstd::encoding::{compress_to_vec, CompressionLevel}; +use ruzstd::encoding::CompressionLevel; use serde::{Deserialize, Serialize}; use shared::error::{str_to_io_error, to_io_error}; use std::fs::File; @@ -40,6 +40,7 @@ fn u32_from_bytes(bytes: &[u8]) -> io::Result { } +#[inline] fn get_entry_index_upper_bound(keys: &[K], key: &K) -> usize where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, @@ -58,15 +59,14 @@ where } -fn query_tree_le(file: &mut R, key: &K) -> Option +fn query_tree_le(file: &mut R, buffer: &mut Vec, key: &K) -> Option where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { let mut offset = 0; - let mut buffer = vec![0u8; BLOCK_SIZE]; loop { - match BPlusTreeNode::::deserialize_from_block(file, &mut buffer, offset, false) { + match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { Ok((node, pointers)) => { if node.is_leaf { let idx = get_entry_index_upper_bound::(&node.keys, key); @@ -272,131 +272,156 @@ where self.children.iter().for_each(|child| child.traverse(visit)); } - fn serialize_to_block(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { + fn serialize_to_block( + &self, + file: &mut W, + buffer: &mut Vec, // must be length BLOCK_SIZE + offset: u64, + ) -> io::Result { + // Keep backward-compatible on-disk layout with minimal allocations. let mut current_offset = offset; + + // zero the working block and set node type buffer.fill(0_u8); let buffer_slice = &mut buffer[..]; - - // Write node type (leaf or internal) buffer_slice[0] = u8::from(self.is_leaf); let mut write_pos = FLAG_SIZE; - // Serialize and write keys + // ---- Write keys (length + bytes) into the first block ---- let keys_encoded = bincode_serialize(&self.keys)?; - let keys_bytes_len = keys_encoded.len(); - buffer_slice[write_pos..write_pos + LEN_SIZE].copy_from_slice(&(u32::try_from(keys_bytes_len).map_err(to_io_error)?).to_le_bytes()); + let keys_len = keys_encoded.len(); + buffer_slice[write_pos..write_pos + LEN_SIZE] + .copy_from_slice(&u32::try_from(keys_len).map_err(to_io_error)?.to_le_bytes()); write_pos += LEN_SIZE; - buffer_slice[write_pos..write_pos + keys_bytes_len].copy_from_slice(&keys_encoded); - write_pos += keys_bytes_len; + // NOTE: By design of the legacy layout, keys are expected to fit into the first block. + buffer_slice[write_pos..write_pos + keys_len].copy_from_slice(&keys_encoded); + write_pos += keys_len; + drop(keys_encoded); - let mut remaining: Option> = None; - // If leaf, serialize and write values + // Prepare pointer offset for internal nodes (must be within first block) + let pointer_offset_within_first_block = if self.is_leaf { 0u64 } else { offset + write_pos as u64 }; + + // ---- Leaf values (optional) ---- if self.is_leaf { + // Encode values and decide compression exactly like the old layout let values_encoded = bincode_serialize(&self.values)?; let use_compression = values_encoded.len() + write_pos >= BLOCK_SIZE; - let compression_byte = if use_compression { 1u8.to_le_bytes() } else { 0u8.to_le_bytes() }; - buffer_slice[write_pos..=write_pos].copy_from_slice(&compression_byte); + + // Compression flag + buffer_slice[write_pos] = u8::from(use_compression); write_pos += FLAG_SIZE; + + // Content bytes (possibly compressed) let content_bytes = if use_compression { - { - // use ruzstd::io::Read; - compress_to_vec(&*values_encoded, CompressionLevel::Fastest) - } + // ruzstd expects a Read implementor; wrap the slice in a Cursor + ruzstd::encoding::compress_to_vec(std::io::Cursor::new(values_encoded.as_slice()), CompressionLevel::Fastest) } else { values_encoded }; - let values_bytes_len = content_bytes.len(); - buffer_slice[write_pos..write_pos + LEN_SIZE].copy_from_slice(&(u32::try_from(values_bytes_len).map_err(to_io_error)?).to_le_bytes()); + + // Write content length + let content_len = content_bytes.len(); + buffer_slice[write_pos..write_pos + LEN_SIZE] + .copy_from_slice(&u32::try_from(content_len).map_err(to_io_error)?.to_le_bytes()); write_pos += LEN_SIZE; - let mut value_bytes_to_write = values_bytes_len; - let bytes_to_write = write_pos + values_bytes_len; - if bytes_to_write > BLOCK_SIZE { - value_bytes_to_write = BLOCK_SIZE - write_pos; - remaining = Some(content_bytes[value_bytes_to_write..].to_vec()); + // Copy as many content bytes as fit into the first block + let space_left = BLOCK_SIZE.saturating_sub(write_pos); + let first_copy = std::cmp::min(space_left, content_len); + buffer_slice[write_pos..write_pos + first_copy] + .copy_from_slice(&content_bytes[..first_copy]); + + // Write the full first block + file.seek(SeekFrom::Start(offset))?; + file.write_all(&buffer_slice[..BLOCK_SIZE])?; + current_offset += BLOCK_SIZE as u64; + + // Stream remaining content (if any) block-by-block without extra allocations + let mut pos = first_copy; + while pos < content_len { + let remaining = content_len - pos; + let chunk = std::cmp::min(remaining, BLOCK_SIZE); + // Copy the next chunk + buffer_slice[..chunk].copy_from_slice(&content_bytes[pos..pos + chunk]); + // If this is a partial block, zero the tail only once + if chunk < BLOCK_SIZE { + buffer_slice[chunk..BLOCK_SIZE].fill(0u8); + } + file.write_all(&buffer_slice[..BLOCK_SIZE])?; // always write full block (pad with zeros) + current_offset += BLOCK_SIZE as u64; + pos += chunk; } - buffer_slice[write_pos..write_pos + value_bytes_to_write].copy_from_slice(&content_bytes[..value_bytes_to_write]); - // write_pos += value_bytes_to_write; + + // Free content buffer ASAP + drop(content_bytes); + } else { + // Internal node: write out the full first block now + file.seek(SeekFrom::Start(offset))?; + file.write_all(&buffer_slice[..BLOCK_SIZE])?; + current_offset += BLOCK_SIZE as u64; } - // Write the complete buffer to file, do not optimize to real filled size, - file.seek(SeekFrom::Start(offset))?; - file.write_all(&buffer_slice[..BLOCK_SIZE])?; // use BLOCK_SIZE - current_offset += BLOCK_SIZE as u64; - - if let Some(mut left_over) = remaining { - // we calculate the needed blocks - let left_over_len = left_over.len(); - if !left_over_len.is_multiple_of(BLOCK_SIZE) { - let padding = BLOCK_SIZE - (left_over_len % BLOCK_SIZE); - left_over.extend(vec![0u8; padding]); - } - let total_len = left_over.len(); - file.write_all(&left_over[..total_len])?; - current_offset += total_len as u64; - } - - // we need to write the pointers after the children are persisted, - // because we need the offsets, which we get after persisting. + // ---- Internal node pointers: serialize children, then write pointer list into first block ---- if !self.is_leaf { - let pointer_offset = offset + write_pos as u64; - let mut pointer = Vec::with_capacity(self.children.len()); + let pointer_offset = pointer_offset_within_first_block; // inside the node's first block + let mut pointer_vec: Vec = Vec::with_capacity(self.children.len()); for child in &self.children { - pointer.push(current_offset); + pointer_vec.push(current_offset); current_offset = child.serialize_to_block(file, buffer, current_offset)?; } - - let pointer_encoded = bincode_serialize(&pointer)?; - let pointer_bytes_len = u32::try_from(pointer_encoded.len()).map_err(to_io_error)?; - + let pointer_encoded = bincode_serialize(&pointer_vec)?; + let pointer_len = u32::try_from(pointer_encoded.len()).map_err(to_io_error)?; file.seek(SeekFrom::Start(pointer_offset))?; - file.write_all(&pointer_bytes_len.to_le_bytes())?; + file.write_all(&pointer_len.to_le_bytes())?; file.write_all(&pointer_encoded)?; } Ok(current_offset) } - fn deserialize_from_block(file: &mut R, buffer: &mut Vec, offset: u64, nested: bool) -> io::Result<(Self, Option>)> { + fn deserialize_from_block( + file: &mut R, + buffer: &mut Vec, + offset: u64, + nested: bool, + ) -> io::Result<(Self, Option>)> { + // Read the full first block into buffer (always aligned on-disk) file.seek(SeekFrom::Start(offset))?; file.read_exact(buffer)?; - // Read the node type directly from buffer + // Node type let is_leaf = buffer[0] == 1u8; let mut read_pos = FLAG_SIZE; - // Deserialize keys + // ---- Keys ---- let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize; read_pos += LEN_SIZE; let keys: Vec = bincode_deserialize(&buffer[read_pos..read_pos + keys_length])?; read_pos += keys_length; - // Deserialize values if leaf node - let values = if is_leaf { - let use_compression = u8::from_le_bytes(buffer[read_pos..=read_pos].try_into().unwrap_or([0u8])) == 1; + // ---- Values for leaf nodes ---- + let values: Vec = if is_leaf { + // compression flag + let use_compression = buffer[read_pos] == 1u8; read_pos += FLAG_SIZE; + + // content length let values_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize; read_pos += LEN_SIZE; + // Fast path: everything is inside the first block let bytes_available_on_block = BLOCK_SIZE - read_pos; - let content_bytes = if values_length > bytes_available_on_block { - let mut left_over_bytes = values_length; - let mut content_chunk = Vec::from(&buffer[read_pos..read_pos + bytes_available_on_block]); - left_over_bytes -= bytes_available_on_block; - while left_over_bytes > 0 { - file.read_exact(buffer)?; - let bytes_to_read = if left_over_bytes >= BLOCK_SIZE { - BLOCK_SIZE - } else { - left_over_bytes - }; - content_chunk.extend(&buffer[..bytes_to_read]); - left_over_bytes -= bytes_to_read; - } - content_chunk[..values_length].to_vec() + let mut content_bytes: Vec = vec![0u8; values_length]; + if values_length <= bytes_available_on_block { + content_bytes[..values_length] + .copy_from_slice(&buffer[read_pos..read_pos + values_length]); } else { - buffer[read_pos..read_pos + values_length].to_vec() - }; + // Copy what's left in the first block + content_bytes[..bytes_available_on_block] + .copy_from_slice(&buffer[read_pos..read_pos + bytes_available_on_block]); + // Read the remaining bytes directly into the final buffer + file.read_exact(&mut content_bytes[bytes_available_on_block..])?; + } let values_bytes = if use_compression { decode_content(&content_bytes).unwrap_or(content_bytes) @@ -404,32 +429,27 @@ where content_bytes }; - let values: Vec = bincode_deserialize(&values_bytes)?; - read_pos += values_length; - values + bincode_deserialize(&values_bytes)? } else { - vec![] + Vec::new() }; - // Deserialize children indices if internal node - let (children, children_pointer) = if is_leaf { - (vec![], None) + // ---- Pointers for internal nodes ---- + let (children, children_pointer): (Vec, Option>) = if is_leaf { + (Vec::new(), None) } else { let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + LEN_SIZE])? as usize; read_pos += LEN_SIZE; let pointers: Vec = bincode_deserialize(&buffer[read_pos..read_pos + pointers_length])?; if nested { - let nodes: Result, io::Error> = pointers - .iter() - .map(|pointer| { - Self::deserialize_from_block(file, buffer, *pointer, nested) - .map(|(node, _)| node) - }) - .collect(); - - (nodes?, None) + let mut nodes = Vec::with_capacity(pointers.len()); + for &ptr in &pointers { + let (child, _) = Self::deserialize_from_block(file, buffer, ptr, nested)?; + nodes.push(child); + } + (nodes, None) } else { - (vec![], Some(pointers)) + (Vec::new(), Some(pointers)) } }; @@ -437,8 +457,8 @@ where } } -fn decode_content(content_bytes: &Vec) -> Option> { - if let Ok(mut decoder) = StreamingDecoder::new(&**content_bytes) { +fn decode_content(content_bytes: &[u8]) -> Option> { + if let Ok(mut decoder) = StreamingDecoder::new(content_bytes) { let mut result = Vec::with_capacity(content_bytes.len()); if decoder.read_to_end(&mut result).is_ok() { return Some(result); @@ -592,15 +612,14 @@ where } } -fn query_tree(file: &mut R, key: &K) -> Option +fn query_tree(file: &mut R, buffer: &mut Vec, key: &K) -> Option where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { let mut offset = 0; - let mut buffer = vec![0u8; BLOCK_SIZE]; loop { - match BPlusTreeNode::::deserialize_from_block(file, &mut buffer, offset, false) { + match BPlusTreeNode::::deserialize_from_block(file, buffer, offset, false) { Ok((node, pointers)) => { if node.is_leaf { return match node.keys.binary_search(key) { @@ -655,6 +674,7 @@ where /// pub struct BPlusTreeQuery { file: BufReader, + buffer: Vec, _marker_k: PhantomData, _marker_v: PhantomData, } @@ -668,6 +688,7 @@ where let file = is_file_valid(file)?; Ok(Self { file: utils::file_reader(file), + buffer: vec![0u8; BLOCK_SIZE], _marker_k: PhantomData, _marker_v: PhantomData, }) @@ -679,7 +700,7 @@ where } pub fn query(&mut self, key: &K) -> Option { - query_tree(&mut self.file, key) + query_tree(&mut self.file, &mut self.buffer, key) } /// On-disk: find largest key <= `key` and return owned V (cloned/deserialized) @@ -692,7 +713,7 @@ where // if seek fails, still try to query — but bail out with None return None; } - query_tree_le(file, key) + query_tree_le(file, &mut self.buffer, key) } // pub fn traverse(&mut self, mut visit: F) @@ -705,6 +726,8 @@ where pub struct BPlusTreeUpdate { file: File, + read_buffer: Vec, + write_buffer: Vec, _marker_k: PhantomData, _marker_v: PhantomData, } @@ -721,6 +744,8 @@ where let file = is_file_valid(utils::open_read_write_file(filepath)?)?; Ok(Self { file, + read_buffer: vec![0u8; BLOCK_SIZE], + write_buffer: vec![0u8; BLOCK_SIZE], _marker_k: PhantomData, _marker_v: PhantomData, }) @@ -728,22 +753,20 @@ where pub fn query(&mut self, key: &K) -> Option { let mut reader = utils::file_reader(&mut self.file); - query_tree(&mut reader, key) + query_tree(&mut reader, &mut self.read_buffer, key) } fn serialize_node(&mut self, offset: u64, node: &BPlusTreeNode) -> io::Result { - let mut buffer = vec![0u8; BLOCK_SIZE]; - let result = node.serialize_to_block(&mut self.file, &mut buffer, offset); + let result = node.serialize_to_block(&mut self.file, &mut self.write_buffer, offset); self.file.flush()?; result } pub fn update(&mut self, key: &K, value: V) -> io::Result { let mut offset = 0; - let mut buffer = vec![0u8; BLOCK_SIZE]; let mut reader = utils::file_reader(&mut self.file); loop { - match BPlusTreeNode::::deserialize_from_block(&mut reader, &mut buffer, offset, false) { + match BPlusTreeNode::::deserialize_from_block(&mut reader, &mut self.read_buffer, offset, false) { Ok((mut node, pointers)) => { if node.is_leaf { return match node.keys.binary_search(key) { @@ -776,7 +799,7 @@ where /// On-disk update helper: find largest key <= `key`. pub fn query_le(&mut self, key: &K) -> Option { let mut reader = utils::file_reader(&mut self.file); - query_tree_le(&mut reader, key) + query_tree_le(&mut reader, &mut self.read_buffer, key) } } diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index a1872015b..baeabe6ae 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -34,8 +34,6 @@ pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path) -> Re // EPG Content epg.write_to_async(&mut writer).await.map_err(|e| notify_err!(format!("failed to write epg: {}", e)))?; - let inner = writer.get_mut(); // Zugriff auf den BufWriter - inner.flush().await.map_err(|e| notify_err!(format!("failed to flush epg: {}", e)))?; debug_if_enabled!("Epg for target {} written to {}", target.name, path.to_str().unwrap_or("?")); Ok(()) diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index d59fc75c0..f53477cd4 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -1,10 +1,9 @@ use std::borrow::Cow; use std::collections::HashSet; use std::fs::{File, OpenOptions}; -use std::io::{BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use std::{env, fs}; - +use std::io::Read as IORead; use shared::error::str_to_io_error; use crate::utils::debug_if_enabled; use shared::utils::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, MAPPING_FILE, SOURCE_FILE, USER_FILE}; @@ -12,20 +11,30 @@ use log::{debug, error}; use path_clean::PathClean; use tokio::fs as tokio_fs; -pub fn file_writer(w: W) -> BufWriter +const WRITER_BUFFER_SIZE: usize = 131_072; // 128kb + +pub fn file_writer(w: W) -> std::io::BufWriter where - W: Write, + W: std::io::Write, { - BufWriter::with_capacity(131_072, w) + std::io::BufWriter::with_capacity(WRITER_BUFFER_SIZE, w) } -pub fn file_reader(r: R) -> BufReader +pub fn file_reader(r: R) -> std::io::BufReader where - R: Read, + R: std::io::Read, { - BufReader::with_capacity(131_072, r) + std::io::BufReader::with_capacity(WRITER_BUFFER_SIZE, r) } +pub fn async_file_writer(w: W) -> tokio::io::BufWriter +where + W: tokio::io::AsyncWrite, +{ + tokio::io::BufWriter::with_capacity(WRITER_BUFFER_SIZE, w) +} + + pub fn get_exe_path() -> PathBuf { let default_path = std::path::PathBuf::from("./"); let current_exe = std::env::current_exe(); From 1193fc2265b9a502c937bc01f397e52aaa6c4101 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 15:27:48 +0100 Subject: [PATCH 03/11] write operations optimized for less memory usage --- Cargo.lock | 6 +- backend/Cargo.toml | 2 +- backend/src/api/api_utils.rs | 7 +- backend/src/api/endpoints/download_api.rs | 6 +- backend/src/api/endpoints/xmltv_api.rs | 6 +- backend/src/model/xmltv.rs | 3 +- backend/src/processing/parser/xmltv.rs | 3 +- .../src/processing/processor/xtream_series.rs | 6 +- .../src/processing/processor/xtream_vod.rs | 4 +- backend/src/repository/epg_repository.rs | 48 +++++--- backend/src/repository/indexed_document.rs | 113 +++++++++++------- .../src/repository/m3u_playlist_iterator.rs | 79 +++++++----- backend/src/repository/m3u_repository.rs | 16 ++- backend/src/repository/playlist_repository.rs | 1 - backend/src/repository/storage.rs | 3 +- backend/src/repository/strm_repository.rs | 24 +++- .../repository/xtream_playlist_iterator.rs | 49 ++++---- backend/src/repository/xtream_repository.rs | 11 +- .../compression/compressed_file_reader.rs | 2 +- .../compressed_file_reader_async.rs | 22 +--- backend/src/utils/file/config_reader.rs | 4 +- backend/src/utils/file/csv_input_reader.rs | 10 +- backend/src/utils/file/file_utils.rs | 9 +- backend/src/utils/geoip.rs | 3 +- backend/src/utils/json_utils.rs | 4 +- backend/src/utils/network/request.rs | 21 ++-- frontend/Cargo.toml | 4 +- shared/Cargo.toml | 2 +- shared/src/utils/string_utils.rs | 39 ++++++ 29 files changed, 320 insertions(+), 187 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2916ee3b3..cceb3baf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1096,7 +1096,7 @@ dependencies = [ [[package]] name = "frontend" -version = "3.2.17" +version = "3.2.18" dependencies = [ "anyhow", "base64", @@ -3793,7 +3793,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.2.17" +version = "3.2.18" dependencies = [ "base64", "bitflags 2.10.0", @@ -4356,7 +4356,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.2.17" +version = "3.2.18" dependencies = [ "arc-swap", "async-compression", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index bc9f6cd94..cbba13ba1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tuliprox" -version = "3.2.17" +version = "3.2.18" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index 376e836be..88615182d 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -12,7 +12,7 @@ use crate::api::model::{ProviderAllocation, ProviderConfig, ProviderStreamState, use crate::model::{ConfigInput, ResourceRetryConfig}; use crate::model::{ConfigTarget, ProxyUserCredentials}; use crate::tools::lru_cache::LRUResourceCache; -use crate::utils::create_new_file_for_write; +use crate::utils::{async_file_reader, async_file_writer, create_new_file_for_write}; use crate::utils::request; use crate::utils::{debug_if_enabled, trace_if_enabled}; use crate::BUILD_TIMESTAMP; @@ -35,7 +35,6 @@ use std::collections::HashMap; use std::path::Path; use std::sync::Arc; use std::time::Duration; -use tokio::io::BufWriter; use tokio::sync::Mutex; use url::Url; @@ -155,7 +154,7 @@ pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoRes match tokio::fs::File::open(file_path).await { Ok(file) => { - let reader = tokio::io::BufReader::new(file); + let reader = async_file_reader(file); let stream = tokio_util::io::ReaderStream::new(reader); let body = axum::body::Body::from_stream(stream); @@ -1107,7 +1106,7 @@ async fn build_stream_response( match create_new_file_for_write(&resource_path).await { Ok(file) => { debug!("Persisting resource stream {sanitized_resource_url} to {}", resource_path.display()); - let writer = BufWriter::new(file); + let writer = async_file_writer(file); let add_cache_content = get_add_cache_content(resource_url, &app_state.cache); let tee = tee_stream(byte_stream, writer, &resource_path, add_cache_content); return try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(tee))); diff --git a/backend/src/api/endpoints/download_api.rs b/backend/src/api/endpoints/download_api.rs index ff69feec6..618d3cc20 100644 --- a/backend/src/api/endpoints/download_api.rs +++ b/backend/src/api/endpoints/download_api.rs @@ -1,7 +1,7 @@ use crate::api::model::AppState; use crate::api::model::{DownloadQueue, FileDownload, FileDownloadRequest}; use crate::model::{AppConfig, VideoDownloadConfig}; -use crate::utils::{async_file_writer, request}; +use crate::utils::{async_file_writer, request, WRITER_BUFFER_SIZE}; use tokio::sync::RwLock; use futures::stream::TryStreamExt; use log::info; @@ -36,8 +36,8 @@ async fn download_file(active: Arc>>, client: &reqwe if let Some(chunk) = item { match buf_writer.write_all(&chunk).await { Ok(()) => { - write_counter += 1; - if write_counter > 50 { + write_counter += chunk.len(); + if write_counter >= WRITER_BUFFER_SIZE { buf_writer.flush().await.map_err(|err| err.to_string())?; write_counter = 0; } diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index 2599281a2..8369651d2 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -8,7 +8,7 @@ use crate::repository::storage::get_target_storage_path; use crate::repository::storage_const; use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path}; use crate::utils; -use crate::utils::{deobscure_text, obscure_text}; +use crate::utils::{async_file_reader, deobscure_text, obscure_text}; use axum::response::IntoResponse; use chrono::{DateTime, Duration, FixedOffset, NaiveDateTime, Offset, TimeZone, Utc}; use chrono_tz::Tz; @@ -206,7 +206,7 @@ async fn serve_epg_with_rewrites( let rewrite_base_url = base_url.to_owned(); match tokio::fs::File::open(epg_path).await { Ok(file) => { - let reader = tokio::io::BufReader::new(file); + let reader = async_file_reader(file); let (tx, rx) = tokio::io::duplex(8192); tokio::spawn(async move { let mut encoder = async_compression::tokio::write::GzipEncoder::new(tx); @@ -220,7 +220,7 @@ async fn serve_epg_with_rewrites( error!("EPG: Failed to write epg doc type {err}"); } - let mut xml_reader = quick_xml::reader::Reader::from_reader(tokio::io::BufReader::new(reader)); + let mut xml_reader = quick_xml::reader::Reader::from_reader(async_file_reader(reader)); let mut xml_writer = quick_xml::writer::Writer::new(encoder); // TODO howto avoid BytesText to escape the doctype "xmltv.dtd" which is written as "e;xmltv.dtd"e; diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index 7896246b3..5ec19fc6c 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -12,6 +12,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use url::Url; use shared::utils::sanitize_sensitive_info; use crate::api::model::AppState; +use crate::utils::async_file_reader; use crate::utils::request::{get_remote_content_as_stream}; pub const EPG_TAG_TV: &str = "tv"; @@ -247,7 +248,7 @@ pub fn get_attr_value(attr: &quick_xml::events::attributes::Attribute) -> Option #[allow(clippy::too_many_lines)] async fn parse_xmltv_for_web_ui(reader: R) -> Result { - let mut reader = quick_xml::reader::Reader::from_reader(tokio::io::BufReader::new(reader)); + let mut reader = quick_xml::reader::Reader::from_reader(async_file_reader(reader)); let mut buf = Vec::new(); let mut channels = Vec::new(); diff --git a/backend/src/processing/parser/xmltv.rs b/backend/src/processing/parser/xmltv.rs index b94dcd537..8cf35cd5e 100644 --- a/backend/src/processing/parser/xmltv.rs +++ b/backend/src/processing/parser/xmltv.rs @@ -13,6 +13,7 @@ use std::collections::HashMap; use std::mem; use std::sync::{Mutex}; use tokio::io::AsyncRead; +use crate::utils::async_file_reader; use crate::utils::compressed_file_reader_async::CompressedFileReaderAsync; /// Splits a string at the first delimiter if the prefix matches a known country code. @@ -412,7 +413,7 @@ where F: FnMut(XmlTag), { let mut stack: Vec = vec![]; - let mut xml_reader = quick_xml::reader::Reader::from_reader(tokio::io::BufReader::new(content)); + let mut xml_reader = quick_xml::reader::Reader::from_reader(async_file_reader(content)); let mut buf = Vec::::new(); loop { match xml_reader.read_event_into_async(&mut buf).await { diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 158388dd8..79c3b1aef 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -96,7 +96,8 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, write_counter += 1; // periodic flush to bound BufWriter memory - if write_counter.is_multiple_of(FLUSH_INTERVAL) { + if write_counter >= FLUSH_INTERVAL { + write_counter = 0; if let Err(err) = content_writer.flush() { errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); } @@ -202,7 +203,8 @@ async fn process_series_info( } write_counter +=1; // periodic flush to bound BufWriter memory - if write_counter.is_multiple_of(FLUSH_INTERVAL) { + if write_counter <= FLUSH_INTERVAL { + write_counter = 0; if let Err(err) = wal_writer.flush() { errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); } diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 8880fb509..a9189f72f 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -109,9 +109,9 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Clie processed_info_ids.insert(provider_id, ts); content_updated = true; write_counter += 1; - // periodic flush to bound BufWriter memory - if write_counter.is_multiple_of(FLUSH_INTERVAL) { + if write_counter >= FLUSH_INTERVAL { + write_counter = 0; if let Err(err) = content_writer.flush() { errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); } diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index baeabe6ae..3903fdfea 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -2,40 +2,56 @@ use crate::model::Epg; use crate::model::{Config, ConfigTarget, TargetOutput}; use crate::repository::m3u_repository::m3u_get_epg_file_path; use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path}; -use crate::utils::debug_if_enabled; +use crate::utils::{async_file_writer, debug_if_enabled}; use shared::error::{notify_err, TuliproxError}; use std::path::Path; use tokio::io::AsyncWriteExt; -// Due to an error in quick_xml we cant write doc type through event. The quotes are escaped and the xml file is invalid. +const XML_PREAMBLE: &str = r#" + +"#; + +// Due to a bug in quick_xml we cannot write the DOCTYPE via event; quotes are escaped and the XML becomes invalid. +// Keep the manual header/doctype write workaround below. // -// // XML Header +// // XML Header via events (DO NOT USE, kept for documentation): // writer.write_event_async(quick_xml::events::Event::Decl(quick_xml::events::BytesDecl::new("1.0", Some("utf-8"), None))) // .await.map_err(|e| notify_err!(format!("failed to write XML header: {}", e)))?; // -// // DOCTYPE +// // DOCTYPE via events (DO NOT USE): // writer.write_event_async(quick_xml::events::Event::DocType(quick_xml::events::BytesText::new(r#"tv SYSTEM "xmltv.dtd""#))) // .await.map_err(|e| notify_err!(format!("failed to write doctype: {}", e)))?; pub async fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path) -> Result<(), TuliproxError> { let file = tokio::fs::File::create(path).await - .map_err(|e| notify_err!(format!("failed to create epg file: {}", e)))?; - let mut buf_writer = tokio::io::BufWriter::new(file); + .map_err(|e| notify_err!(format!("failed to create epg file {}: {}", path.display(), e)))?; - // Work-Around BytesText DocType escape, see below - buf_writer.write_all(b"\n").await - .map_err(|e| notify_err!(format!("failed to write XML header: {}", e)))?; + // Use a larger buffer for sequential writes to reduce syscalls + let mut buf_writer = async_file_writer(file); - buf_writer.write_all(b"\n").await - .map_err(|e| notify_err!(format!("failed to write doctype: {}", e)))?; + // Header/DOCTYPE workaround: write both lines in a single call for fewer syscalls + buf_writer + .write_all(XML_PREAMBLE.as_bytes()) + .await + .map_err(|e| notify_err!(format!("failed to write XML preamble {}: {}", path.display(), e)))?; + // EPG content streamed via quick_xml writer (compact output) let mut writer = quick_xml::writer::Writer::new(buf_writer); + epg + .write_to_async(&mut writer) + .await + .map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?; - // EPG Content - epg.write_to_async(&mut writer).await.map_err(|e| notify_err!(format!("failed to write epg: {}", e)))?; + // Ensure buffers are flushed to the OS and capture any I/O error + let mut buf_writer = writer.into_inner(); + buf_writer + .flush() + .await + .map_err(|e| notify_err!(format!("failed to flush epg {}: {}", path.display(), e)))?; + buf_writer.shutdown().await.map_err(|e| notify_err!(format!("failed to write epg {}: {}", path.display(), e)))?; - debug_if_enabled!("Epg for target {} written to {}", target.name, path.to_str().unwrap_or("?")); + debug_if_enabled!("Epg for target {} written to {}", target.name, path.display()); Ok(()) } @@ -46,7 +62,7 @@ pub async fn epg_write(cfg: &Config, target: &ConfigTarget, target_path: &Path, match xtream_get_storage_path(cfg, &target.name) { Some(path) => { let epg_path = xtream_get_epg_file_path(&path); - debug_if_enabled!("writing xtream epg to {}", epg_path.to_str().unwrap_or("?")); + debug_if_enabled!("writing xtream epg to {}", epg_path.display()); epg_write_file(target, epg_data, &epg_path).await?; } None => return Err(notify_err!(format!("failed to serialize epg for target: {}, storage path not found", target.name))), @@ -54,7 +70,7 @@ pub async fn epg_write(cfg: &Config, target: &ConfigTarget, target_path: &Path, } TargetOutput::M3u(_) => { let path = m3u_get_epg_file_path(target_path); - debug_if_enabled!("writing m3u epg to {}", path.to_str().unwrap_or("?")); + debug_if_enabled!("writing m3u epg to {}", path.display()); epg_write_file(target, epg_data, &path).await?; } TargetOutput::Strm(_) | TargetOutput::HdHomeRun(_) => {} diff --git a/backend/src/repository/indexed_document.rs b/backend/src/repository/indexed_document.rs index e03a1fa22..b0ab04eee 100644 --- a/backend/src/repository/indexed_document.rs +++ b/backend/src/repository/indexed_document.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use tempfile::NamedTempFile; use shared::error::{str_to_io_error, to_io_error}; use crate::utils; -use crate::utils::{bincode_deserialize, bincode_serialize}; +use crate::utils::{bincode_deserialize, bincode_serialize, WRITER_BUFFER_SIZE}; const BLOCK_SIZE: usize = 4096; const LEN_SIZE: usize = 4; @@ -163,54 +163,67 @@ where where T: ?Sized + serde::Serialize, { - let encoded_bytes = bincode_serialize(doc).map_err(|_| Error::new(ErrorKind::InvalidData, "Failed to serialize document"))?; - let mut new_record_appended = false; // do i need to change the index and set the new offset + // Determine target position (overwrite in place or append) if let Some(&offset) = self.index_tree.query(&doc_id) { + // Existing entry: check size and choose strategy self.main_file.seek(SeekFrom::Start(u64::from(offset)))?; - let size = IndexedDocument::read_content_size(&mut self.main_file)?; - if size == encoded_bytes.len() { - // check if it is equal - let mut record_buffer = vec![0; size]; - // record_buffer.resize(size, 0); + let old_size = IndexedDocument::read_content_size(&mut self.main_file)?; - self.main_file.read_exact(&mut record_buffer)?; - if record_buffer == encoded_bytes { - return Ok(()); - } - } + // Prepare new payload (bincode utils, compatible with project) + let encoded = bincode_serialize(doc) + .map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?; + let new_size: SizeType = SizeType::try_from(encoded.len() as u64) + .map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?; - if encoded_bytes.len() > size { - // does not fit we need to append, file is fragmented + if usize::try_from(new_size).unwrap_or(usize::MAX) > old_size { + // Does not fit -> append at end of file (mark fragmentation) if !self.fragmented { self.fragmented = true; IndexedDocument::write_fragmentation(&mut self.main_file, true)?; } self.main_file.seek(SeekFrom::End(0))?; - new_record_appended = true; - } else { - self.main_file.seek(SeekFrom::Start(u64::from(offset)))?; - } - } else { - self.main_file.seek(SeekFrom::End(0))?; - new_record_appended = true; - } + // We write: length + payload + self.dirty = true; + self.main_file.write_all(&new_size.to_le_bytes())?; + self.main_file.write_all(&encoded)?; + self.index_tree.insert(doc_id, self.main_offset); + // Increase main_offset: LEN_SIZE + new_size + let written_bytes_u64 = (LEN_SIZE as u64) + u64::from(new_size); + let written_bytes: SizeType = SizeType::try_from(written_bytes_u64) + .map_err(|e| str_to_io_error(&format!("Written byte count overflow for {}: {e}", self.main_path.display())))?; + self.main_offset = self.main_offset.checked_add(written_bytes) + .ok_or_else(|| str_to_io_error(&format!("main_offset overflow while appending to {}", self.main_path.display())))?; + return Ok(()); + } + // Fits (<=): overwrite at the same position + self.main_file.seek(SeekFrom::Start(u64::from(offset)))?; + self.dirty = true; + self.main_file.write_all(&new_size.to_le_bytes())?; + self.main_file.write_all(&encoded)?; + return Ok(()); + } + // New entry -> append + self.main_file.seek(SeekFrom::End(0))?; + // Determine size and write it, then write payload + let encoded = bincode_serialize(doc) + .map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?; + let new_size: SizeType = SizeType::try_from(encoded.len() as u64) + .map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?; self.dirty = true; + self.main_file + .write_all(&new_size.to_le_bytes()) + .map_err(|e| str_to_io_error(&format!("Failed to write length prefix to {}: {e}", self.main_path.display())))?; + self.main_file + .write_all(&encoded) + .map_err(|e| str_to_io_error(&format!("Failed to write document payload to {}: {e}", self.main_path.display())))?; - let encoded_bytes_len = SizeType::try_from(encoded_bytes.len()).map_err(to_io_error)?; - self.main_file.write_all(&encoded_bytes_len.to_le_bytes())?; - match utils::check_write(&self.main_file.write_all(&encoded_bytes)) { - Ok(()) => { - if new_record_appended { - self.index_tree.insert(doc_id, self.main_offset); - let written_bytes = SizeType::try_from(encoded_bytes.len() + LEN_SIZE).map_err(to_io_error)?; - self.main_offset += written_bytes; - } - } - Err(err) => { - return Err(str_to_io_error(&format!("failed to write document: {} - {}", self.main_path.display(), err))); - } - } + self.index_tree.insert(doc_id, self.main_offset); + let written_bytes_u64 = (LEN_SIZE as u64) + u64::from(new_size); + let written_bytes: SizeType = SizeType::try_from(written_bytes_u64) + .map_err(|e| str_to_io_error(&format!("Written byte count overflow for {}: {e}", self.main_path.display())))?; + self.main_offset = self.main_offset.checked_add(written_bytes) + .ok_or_else(|| str_to_io_error(&format!("main_offset overflow while appending to {}", self.main_path.display())))?; Ok(()) } } @@ -237,6 +250,7 @@ where { main_file: BufReader, index_tree: IndexedDocumentIndex, + buffer: Vec, t_type: PhantomData, } @@ -252,8 +266,10 @@ where let index_tree = IndexedDocumentIndex::::load(index_path)?; Ok(Self { - main_file: utils::file_reader(main_file), + // Larger read buffer for sequential access + main_file: BufReader::with_capacity(BLOCK_SIZE * 64, main_file), index_tree, + buffer: Vec::with_capacity(BLOCK_SIZE), t_type: PhantomData, }) } else { @@ -264,9 +280,12 @@ where if let Some(offset) = self.index_tree.query(doc_id) { self.main_file.seek(SeekFrom::Start(u64::from(*offset)))?; let buf_size = IndexedDocument::read_content_size(&mut self.main_file)?; - let mut buffer: Vec = vec![0; buf_size]; - self.main_file.read_exact(&mut buffer)?; - if let Ok(item) = bincode_deserialize::(&buffer) { + if self.buffer.capacity() < buf_size { + self.buffer.reserve(buf_size - self.buffer.capacity()); + } + self.buffer.resize(buf_size, 0u8); + self.main_file.read_exact(&mut self.buffer[..buf_size])?; + if let Ok(item) = bincode_deserialize::(&self.buffer[..buf_size]) { return Ok(item); } } @@ -308,7 +327,8 @@ where Ok(file) => { Ok(Self { main_path: main_path.to_path_buf(), - main_file: utils::file_reader(file), + // Larger read buffer for sequential iteration + main_file: BufReader::with_capacity(BLOCK_SIZE * 128, file), offsets, index: 0, failed: false, @@ -467,6 +487,8 @@ where self.index_tree.traverse(|keys, values| { keys.iter().zip(values.iter()).for_each(|(key, &offset)| key_offset.push((key.clone(), offset))); }); + // For better disk locality: sort by offset (sequential reads) + key_offset.sort_by_key(|(_, off)| *off); let mut gc_writer = utils::file_writer(&gc_file); let fragmented_byte = 0u8.to_le_bytes(); @@ -475,6 +497,8 @@ where let mut gc_offset = 1usize; // offset is 1 because of fragment bit let mut buffer: Vec = Vec::with_capacity(BLOCK_SIZE); let mut size_bytes = [0u8; LEN_SIZE]; + let mut write_counter = 0usize; + for (key, offset) in key_offset { // read old content self.main_file.seek(SeekFrom::Start(u64::from(offset)))?; @@ -489,6 +513,11 @@ where gc_writer.write_all(&size_bytes)?; gc_writer.write_all(&buffer[0..buf_size])?; + write_counter += buf_size; + if write_counter >= WRITER_BUFFER_SIZE { + write_counter = 0; + gc_writer.flush()?; + } let pointer = OffsetPointer::try_from(gc_offset).map_err(to_io_error)?; self.index_tree.insert(key, pointer); diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index b0cdd1f3e..dd24600f0 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -10,6 +10,7 @@ use crate::repository::storage_const; use crate::repository::user_repository::user_get_bouquet_filter; use crate::utils::FileReadGuard; use std::collections::HashSet; +// concat_string! macro from shared utils is used for efficient String building #[allow(clippy::struct_excessive_bools)] pub struct M3uPlaylistIterator { @@ -67,26 +68,39 @@ impl M3uPlaylistIterator { } fn get_rewritten_url(&self, m3u_pli: &M3uPlaylistItem, typed: bool, prefix_path: &str) -> String { - if typed { - let stream_type = match m3u_pli.item_type { + // Build URL efficiently with a single allocation using concat_string! macro + let stream_type: &str = if typed { + match m3u_pli.item_type { PlaylistItemType::Live | PlaylistItemType::Catchup | PlaylistItemType::LiveUnknown | PlaylistItemType::LiveHls | PlaylistItemType::LiveDash => "live", PlaylistItemType::Video => "movie", - PlaylistItemType::Series - | PlaylistItemType::SeriesInfo => "series", - }; - format!("{}/{prefix_path}/{stream_type}/{}/{}/{}", - &self.base_url, - &self.username, - &self.password, - m3u_pli.virtual_id + PlaylistItemType::Series | PlaylistItemType::SeriesInfo => "series", + } + } else { + "" + }; + + let mut cap = self.base_url.len() + + prefix_path.len() + + self.username.len() + + self.password.len() + + 32; // separators and id + if typed { cap += stream_type.len() + 1; } + + if typed { + shared::concat_string!( + cap = cap; + &self.base_url, "/", prefix_path, "/", stream_type, "/", + &self.username, "/", &self.password, "/", m3u_pli.virtual_id ) } else { - format!("{}/{prefix_path}/{}/{}/{}", - &self.base_url, &self.username, &self.password, m3u_pli.virtual_id + shared::concat_string!( + cap = cap; + &self.base_url, "/", prefix_path, "/", + &self.username, "/", &self.password, "/", m3u_pli.virtual_id ) } } @@ -101,14 +115,15 @@ impl M3uPlaylistIterator { fn get_next(&mut self) -> Option<(M3uPlaylistItem, bool)> { let entry = if let Some(set) = &self.filter { if let Some((current_item, _)) = self.lookup_item.take() { - let next_valid = self.reader.find(|(pli, _)| set.contains(&pli.group.clone())); + // Avoid cloning strings while filtering + let next_valid = self.reader.find(|(pli, _)| set.contains(pli.group.as_str())); self.lookup_item = next_valid; let has_next = self.lookup_item.is_some(); Some((current_item, has_next)) } else { - let current_item = self.reader.find(|(item, _)| set.contains(&item.group.clone())); + let current_item = self.reader.find(|(item, _)| set.contains(item.group.as_str())); if let Some((item, _)) = current_item { - self.lookup_item = self.reader.find(|(item, _)| set.contains(&item.group.clone())); + self.lookup_item = self.reader.find(|(item, _)| set.contains(item.group.as_str())); let has_next = self.lookup_item.is_some(); Some((item, has_next)) } else { @@ -121,19 +136,29 @@ impl M3uPlaylistIterator { // TODO hls and unknown reverse proxy entry.map(|(mut m3u_pli, has_next)| { - let is_redirect = self.proxy_type.is_redirect(m3u_pli.item_type) || self.target_options.as_ref().and_then(|o| o.force_redirect.as_ref()).is_some_and(|f| f.has_cluster(m3u_pli.item_type)); - let should_rewrite_urls = if is_redirect { self.mask_redirect_url} else { true }; - let rewrite_urls = if should_rewrite_urls { - Some((self.get_stream_url(&m3u_pli, self.include_type_in_url), if self.rewrite_resource { Some(self.get_resource_url(&m3u_pli)) } else { None })) - } else { - None - }; - let url = m3u_pli.url.clone(); - let (stream_url, resource_url) = rewrite_urls - .map_or_else(|| (url, None), |(su, ru)| (su, ru.as_ref().map(String::to_string))); + let is_redirect = self.proxy_type.is_redirect(m3u_pli.item_type) + || self + .target_options + .as_ref() + .and_then(|o| o.force_redirect.as_ref()) + .is_some_and(|f| f.has_cluster(m3u_pli.item_type)); + let should_rewrite_urls = if is_redirect { self.mask_redirect_url } else { true }; + + if should_rewrite_urls { + let stream_url = self.get_stream_url(&m3u_pli, self.include_type_in_url); + let resource_url = if self.rewrite_resource { + Some(self.get_resource_url(&m3u_pli)) + } else { + None + }; + m3u_pli.t_stream_url = stream_url; + m3u_pli.t_resource_url = resource_url; + } else { + // Keep original URL (clone required because target field is distinct) + m3u_pli.t_stream_url = m3u_pli.url.clone(); + m3u_pli.t_resource_url = None; + } - m3u_pli.t_stream_url.clone_from(&stream_url); - m3u_pli.t_resource_url.clone_from(&resource_url); (m3u_pli, has_next) }) } diff --git a/backend/src/repository/m3u_repository.rs b/backend/src/repository/m3u_repository.rs index 7d97e0f3c..97c68384a 100644 --- a/backend/src/repository/m3u_repository.rs +++ b/backend/src/repository/m3u_repository.rs @@ -15,8 +15,9 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use log::error; use tokio::fs; -use tokio::io::{AsyncWriteExt, BufWriter as AsyncBufWriter}; +use tokio::io::{AsyncWriteExt}; use tokio::task; +use crate::utils::{async_file_writer, WRITER_BUFFER_SIZE}; macro_rules! cant_write_result { ($path:expr, $err:expr) => { @@ -53,13 +54,22 @@ async fn persist_m3u_playlist_as_text( let Some(m3u_filename) = utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) else { return Ok(()); }; let file = await_playlist_write!(fs::File::create(&m3u_filename), "Can't write m3u plain playlist {} - {}", m3u_filename.display()); - let mut writer = AsyncBufWriter::new(file); + // Larger buffer for sequential writes to reduce syscalls + let mut writer = async_file_writer(file); await_playlist_write!(writer.write_all(b"#EXTM3U\n"), "Failed to write header to {} - {}", m3u_filename.display()); + let mut write_counter = 0usize; + for m3u in m3u_playlist.iter() { let line = m3u.to_m3u(target.options.as_ref(), false); - await_playlist_write!(writer.write_all(line.as_bytes()), "Failed to write entry to {} - {}", m3u_filename.display()); + let bytes = line.as_bytes(); + await_playlist_write!(writer.write_all(bytes), "Failed to write entry to {} - {}", m3u_filename.display()); await_playlist_write!(writer.write_all(b"\n"), "Failed to write newline to {} - {}", m3u_filename.display()); + write_counter += bytes.len() + 1; + if write_counter >= WRITER_BUFFER_SIZE { + await_playlist_write!(writer.flush(), "Failed to flush {} - {}", m3u_filename.display()); + write_counter = 0; + } } await_playlist_write!(writer.flush(), "Failed to flush {} - {}", m3u_filename.display()); diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 519898ccb..327f4d89a 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -193,7 +193,6 @@ async fn load_m3u_target_storage(app_config: &AppConfig, target: &ConfigTarget) } - pub async fn load_playlists_into_memory_cache(app_state: &AppState) -> Result<(), TuliproxError> { for sources in &app_state.app_config.sources.load().sources { for target in &sources.targets { diff --git a/backend/src/repository/storage.rs b/backend/src/repository/storage.rs index b964b706c..132328f88 100644 --- a/backend/src/repository/storage.rs +++ b/backend/src/repository/storage.rs @@ -6,7 +6,8 @@ use crate::repository::storage_const; use crate::utils; pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { - target_path.join(PathBuf::from(storage_const::FILE_ID_MAPPING)) + // Join directly with &str to avoid an intermediate PathBuf allocation + target_path.join(storage_const::FILE_ID_MAPPING) } pub fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result { diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 0c57af254..f3628b215 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -8,7 +8,7 @@ use crate::repository::storage::{ensure_target_storage_path, get_input_storage_p use crate::repository::storage_const; use crate::repository::xtream_repository::{xtream_get_record_file_path, InputVodInfoRecord}; use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; -use crate::utils::{normalize_string_path, truncate_filename, FileReadGuard}; +use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, FileReadGuard, WRITER_BUFFER_SIZE}; use chrono::Datelike; use filetime::{set_file_times, FileTime}; use log::{error, trace}; @@ -18,7 +18,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::fs::{create_dir_all, remove_dir, remove_file, File}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use shared::model::{ClusterFlags, FieldGetAccessor, PlaylistGroup, PlaylistItem, PlaylistItemType, StrmExportStyle, UUIDType}; use crate::utils; // Import the new MediaQuality struct @@ -932,22 +932,34 @@ async fn write_strm_index_file( let file = File::create(index_file_path) .await .map_err(|err| format!("Failed to create strm index file: {} {err}", index_file_path.display()))?; - let mut writer = BufWriter::new(file); + // Use a larger buffered writer for sequential writes to reduce syscalls + let mut writer = async_file_writer(file); + let mut write_counter = 0usize; let new_line = "\n".as_bytes(); for entry in entries { + let bytes = entry.as_bytes(); + write_counter += bytes.len() + 1; writer - .write_all(entry.as_bytes()) + .write_all(bytes) .await .map_err(|err| format!("Failed to write strm index entry: {err}"))?; writer .write(new_line) .await .map_err(|err| format!("Failed to write strm index entry: {err}"))?; + if write_counter >= WRITER_BUFFER_SIZE { + write_counter = 0; + writer.flush().await.map_err(|err| format!("Failed to flush: {err}"))?; + } } writer .flush() .await .map_err(|err| format!("failed to write strm index entry: {err}"))?; + writer + .shutdown() + .await + .map_err(|err| format!("failed to write strm index entry: {err}"))?; Ok(()) } @@ -989,7 +1001,7 @@ async fn write_strm_file( async fn has_strm_file_same_hash(file_path: &PathBuf, content_hash: UUIDType) -> bool { if let Ok(file) = File::open(&file_path).await { - let mut reader = BufReader::new(file); + let mut reader = async_file_reader(file); let mut buffer = Vec::new(); match reader.read_to_end(&mut buffer).await { Ok(_) => { @@ -1018,7 +1030,7 @@ fn get_credentials_and_server_info( async fn read_strm_file_index(strm_file_index_path: &Path) -> std::io::Result> { let file = File::open(strm_file_index_path).await?; - let reader = BufReader::new(file); + let reader = async_file_reader(file); let mut result = HashSet::new(); let mut lines = reader.lines(); while let Ok(Some(line)) = lines.next_line().await { diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index 122a0c795..8b64d3714 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -1,21 +1,22 @@ -use std::collections::HashSet; -use log::error; -use serde_json::Value; -use shared::model::{TargetType, XtreamCluster, XtreamPlaylistItem}; -use shared::error::info_err; -use shared::error::{TuliproxError}; -use crate::model::{xtream_playlistitem_to_document, AppConfig, ProxyUserCredentials}; -use crate::model::{ConfigTarget}; use crate::model::XtreamMappingOptions; -use crate::repository::indexed_document::{IndexedDocumentIterator}; +use crate::model::{xtream_playlistitem_to_document, AppConfig, ProxyUserCredentials}; +use crate::model::ConfigTarget; +use crate::repository::indexed_document::IndexedDocumentIterator; use crate::repository::user_repository::user_get_bouquet_filter; use crate::repository::xtream_repository::{xtream_get_file_paths, xtream_get_storage_path}; use crate::utils::FileReadGuard; +use log::error; +use serde_json::Value; +use shared::error::info_err; +use shared::error::TuliproxError; +use shared::model::{TargetType, XtreamCluster, XtreamPlaylistItem}; +use std::collections::HashSet; pub struct XtreamPlaylistIterator { reader: IndexedDocumentIterator, options: XtreamMappingOptions, - filter: Option>, + // Use parsed numeric filter to avoid per-item String allocations (no to_string per check) + filter_ids: Option>, base_url: String, user: ProxyUserCredentials, lookup_item: Option<(XtreamPlaylistItem, bool)>, // this is for filtered iteration @@ -49,11 +50,15 @@ impl XtreamPlaylistIterator { let server_info = app_config.get_user_server_info(user); let filter = user_get_bouquet_filter(&config, &user.username, category_id, TargetType::Xtream, cluster).await; + // Parse bouquet filter (strings) once into u32 set to minimize per-item allocations + let filter_ids: Option> = filter.as_ref().map(|set| { + set.iter().filter_map(|s| s.parse::().ok()).collect() + }); Ok(Self { reader, options, - filter, + filter_ids, _file_lock: file_lock, base_url: server_info.get_base_url(), user: user.clone(), @@ -69,16 +74,16 @@ impl XtreamPlaylistIterator { error!("Could not deserialize xtream item: {}", self.reader.get_path().display()); return None; } - if let Some(set) = &self.filter { + if let Some(set) = &self.filter_ids { if let Some((current_item, _)) = self.lookup_item.take() { - let next_valid = self.reader.find(|(pli, _)| set.contains(&pli.category_id.to_string())); + let next_valid = self.reader.find(|(pli, _)| set.contains(&pli.category_id)); self.lookup_item = next_valid; let has_next = self.lookup_item.is_some(); Some((current_item, has_next)) } else { - let current_item = self.reader.find(|(item, _)| set.contains(&item.category_id.to_string())); + let current_item = self.reader.find(|(item, _)| set.contains(&item.category_id)); if let Some((item, _)) = current_item { - self.lookup_item = self.reader.find(|(item, _)| set.contains(&item.category_id.to_string())); + self.lookup_item = self.reader.find(|(item, _)| set.contains(&item.category_id)); let has_next = self.lookup_item.is_some(); Some((item, has_next)) } else { @@ -89,7 +94,6 @@ impl XtreamPlaylistIterator { self.reader.next() } } - } impl Iterator for XtreamPlaylistIterator { @@ -105,12 +109,12 @@ pub struct XtreamPlaylistJsonIterator { } impl XtreamPlaylistJsonIterator { -pub async fn new( - cluster: XtreamCluster, - config: &AppConfig, - target: &ConfigTarget, - category_id: Option, - user: &ProxyUserCredentials, + pub async fn new( + cluster: XtreamCluster, + config: &AppConfig, + target: &ConfigTarget, + category_id: Option, + user: &ProxyUserCredentials, ) -> Result { Ok(Self { inner: XtreamPlaylistIterator::new(cluster, config, target, category_id, user).await? @@ -122,7 +126,6 @@ pub fn to_doc(pli: &XtreamPlaylistItem, url: &str, options: &XtreamMappingOption xtream_playlistitem_to_document(pli, url, options, user) } - impl Iterator for XtreamPlaylistJsonIterator { type Item = (String, bool); fn next(&mut self) -> Option { diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index 6f90ed537..e3952ff8a 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -1076,15 +1076,12 @@ where { match channels { Some((_, chans)) => { - // Convert iterator items to Result + // Convert iterator items to Result with minimal allocations let mapped = chans.map(move |(item, has_next)| { match serde_json::to_string(&item) { - Ok(content) => { - Ok(Bytes::from(if has_next { - format!("{content},") - } else { - content - })) + Ok(mut content) => { + if has_next { content.push(','); } + Ok(Bytes::from(content)) } Err(_) => Ok(Bytes::from("")), } diff --git a/backend/src/utils/compression/compressed_file_reader.rs b/backend/src/utils/compression/compressed_file_reader.rs index 5326c7613..33bc34e22 100644 --- a/backend/src/utils/compression/compressed_file_reader.rs +++ b/backend/src/utils/compression/compressed_file_reader.rs @@ -26,7 +26,7 @@ impl CompressedFileReader { }; Ok(Self { - reader: BufReader::new(reader), + reader: file_reader(reader), }) } } diff --git a/backend/src/utils/compression/compressed_file_reader_async.rs b/backend/src/utils/compression/compressed_file_reader_async.rs index 9ea2b60bd..cc6d7e30f 100644 --- a/backend/src/utils/compression/compressed_file_reader_async.rs +++ b/backend/src/utils/compression/compressed_file_reader_async.rs @@ -6,7 +6,7 @@ use tokio::io::{ self, AsyncRead, BufReader, AsyncSeekExt, AsyncReadExt, ReadBuf, }; use async_compression::tokio::bufread::{GzipDecoder, ZlibDecoder}; - +use crate::utils::async_file_reader; use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; pub struct CompressedFileReaderAsync { @@ -17,7 +17,7 @@ impl CompressedFileReaderAsync { pub async fn new(path: &Path) -> std::io::Result { let file: File = tokio::fs::File::open(path).await?; - let mut buffered_file = BufReader::new(file); + let mut buffered_file = async_file_reader(file); let mut header = [0u8; 2]; buffered_file.read_exact(&mut header).await?; buffered_file.seek(io::SeekFrom::Start(0)).await?; @@ -31,7 +31,7 @@ impl CompressedFileReaderAsync { }; Ok(Self { - reader: BufReader::new(reader), + reader: async_file_reader(reader), }) } } @@ -46,19 +46,3 @@ impl AsyncRead for CompressedFileReaderAsync { } } -// -// impl AsyncBufRead for CompressedFileReaderAsync { -// fn poll_fill_buf( -// self: Pin<&mut Self>, -// cx: &mut Context<'_>, -// ) -> Poll> { -// unsafe { -// let this = self.get_unchecked_mut(); -// Pin::new_unchecked(&mut this.reader).poll_fill_buf(cx) -// } -// } -// -// fn consume(mut self: Pin<&mut Self>, amt: usize) { -// Pin::new(&mut self.reader).consume(amt) -// } -// } diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index c42d2429b..4b6e37feb 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -13,7 +13,7 @@ use shared::model::{ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAlias use shared::utils::{CONSTANTS}; use std::env; use std::fs::File; -use std::io::{self, BufReader, Read}; +use std::io::{self, Read}; use std::path::PathBuf; use std::sync::Arc; use arc_swap::access::{Access}; @@ -40,7 +40,7 @@ pub fn config_file_reader(file: File, resolve_env: bool) -> impl Read if resolve_env { EitherReader::Left(EnvResolvingReader::new(file_reader(file))) } else { - EitherReader::Right(BufReader::new(file)) + EitherReader::Right(file_reader(file)) } } diff --git a/backend/src/utils/file/csv_input_reader.rs b/backend/src/utils/file/csv_input_reader.rs index b64e52f12..17fd6faf6 100644 --- a/backend/src/utils/file/csv_input_reader.rs +++ b/backend/src/utils/file/csv_input_reader.rs @@ -189,7 +189,7 @@ pub fn get_csv_file_path(file_uri: &str) -> Result { #[cfg(test)] mod tests { use crate::utils::file::csv_input_reader::csv_read_inputs_from_reader; - use crate::utils::resolve_env_var; + use crate::utils::{file_reader, resolve_env_var}; use std::io::{BufReader, Cursor}; use shared::model::InputType; @@ -209,7 +209,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34 #[test] fn test_read_inputs_xtream_as_m3u() { - let reader = BufReader::new(Cursor::new(XTREAM_BATCH)); + let reader = file_reader(Cursor::new(XTREAM_BATCH)); let result = csv_read_inputs_from_reader(InputType::M3uBatch, reader); assert!(result.is_ok()); let aliases = result.unwrap(); @@ -221,7 +221,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34 #[test] fn test_read_inputs_m3u_as_m3u() { - let reader = BufReader::new(Cursor::new(M3U_BATCH)); + let reader = file_reader(Cursor::new(M3U_BATCH)); let result = csv_read_inputs_from_reader(InputType::M3uBatch, reader); assert!(result.is_ok()); let aliases = result.unwrap(); @@ -233,7 +233,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34 #[test] fn test_read_inputs_xtream_as_xtream() { - let reader = BufReader::new(Cursor::new(XTREAM_BATCH)); + let reader = file_reader(Cursor::new(XTREAM_BATCH)); let result = csv_read_inputs_from_reader(InputType::XtreamBatch, reader); assert!(result.is_ok()); let aliases = result.unwrap(); @@ -245,7 +245,7 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34 #[test] fn test_read_inputs_m3u_as_xtream() { - let reader = BufReader::new(Cursor::new(M3U_BATCH)); + let reader = file_reader(Cursor::new(M3U_BATCH)); let result = csv_read_inputs_from_reader(InputType::XtreamBatch, reader); assert!(result.is_ok()); let aliases = result.unwrap(); diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index f53477cd4..1e719ded8 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -11,7 +11,7 @@ use log::{debug, error}; use path_clean::PathClean; use tokio::fs as tokio_fs; -const WRITER_BUFFER_SIZE: usize = 131_072; // 128kb +pub const WRITER_BUFFER_SIZE: usize = 256*1024; // 256kb pub fn file_writer(w: W) -> std::io::BufWriter where @@ -34,6 +34,13 @@ where tokio::io::BufWriter::with_capacity(WRITER_BUFFER_SIZE, w) } +pub fn async_file_reader(r: R) -> tokio::io::BufReader +where + R: tokio::io::AsyncRead, +{ + tokio::io::BufReader::with_capacity(WRITER_BUFFER_SIZE, r) +} + pub fn get_exe_path() -> PathBuf { let default_path = std::path::PathBuf::from("./"); diff --git a/backend/src/utils/geoip.rs b/backend/src/utils/geoip.rs index ff3430cda..e5cadd7d1 100644 --- a/backend/src/utils/geoip.rs +++ b/backend/src/utils/geoip.rs @@ -100,13 +100,14 @@ mod test { use std::fs::File; use std::io::BufReader; use std::path::PathBuf; + use crate::utils::file_reader; #[test] pub fn test_csv() { let db_file = PathBuf::from("/projects/m3u-test/asn-country-ipv4.db"); let source = PathBuf::from("/projects/m3u-test/asn-country-ipv4.csv"); let file = File::open(source).expect("Could not open csv file"); - let reader = BufReader::new(file); + let reader = file_reader(file); let mut geo_ip = GeoIp::new(); let _ = geo_ip.import_ipv4_from_csv(reader, &db_file).expect("Could not import csv"); diff --git a/backend/src/utils/json_utils.rs b/backend/src/utils/json_utils.rs index 7ef9e90cc..43888607a 100644 --- a/backend/src/utils/json_utils.rs +++ b/backend/src/utils/json_utils.rs @@ -1,4 +1,4 @@ -use crate::utils::file_reader; +use crate::utils::{file_reader, file_writer}; use serde::Serialize; use serde_json::Value; use std::collections::{HashMap, HashSet}; @@ -49,7 +49,7 @@ where T: Serialize, { let file = std::fs::File::create(path)?; - let mut buf_writer = std::io::BufWriter::new(file); + let mut buf_writer = file_writer(file); serde_json::to_writer(&mut buf_writer, value)?; buf_writer.flush()?; buf_writer.into_inner()?.sync_all() diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 4b15cde7b..3986c0eac 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -8,7 +8,7 @@ use log::{debug, error, log_enabled, trace, Level}; use reqwest::header::CONTENT_ENCODING; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use tokio::fs::File; -use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter}; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio_util::io::StreamReader; use url::Url; @@ -20,7 +20,7 @@ use crate::model::{ConfigInput}; use crate::repository::storage::{get_input_storage_path}; use crate::repository::storage_const; use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; -use crate::utils::{debug_if_enabled}; +use crate::utils::{async_file_reader, async_file_writer, debug_if_enabled, WRITER_BUFFER_SIZE}; use shared::utils::{filter_request_header, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP}; use crate::utils::{get_file_path, persist_file}; @@ -209,7 +209,7 @@ pub async fn get_local_file_content(file_path: &Path) -> Result { if response.status().is_success() { // Open a file in write mode - let mut file = BufWriter::with_capacity(8192, File::create(file_path).await?); + let mut writer = async_file_writer(File::create(file_path).await?); + let mut write_counter = 0; // Stream the response body in chunks let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { match chunk { Ok(bytes) => { - file.write_all(&bytes).await?; + write_counter += bytes.len(); + writer.write_all(&bytes).await?; + if write_counter >= WRITER_BUFFER_SIZE { + writer.flush().await?; + write_counter = 0; + } } Err(err) => { return Err(str_to_io_error(&format!("Failed to read chunk: {err}"))); @@ -262,7 +268,8 @@ async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInpu } } - file.flush().await?; + writer.flush().await?; + writer.shutdown().await?; let elapsed = start_time.elapsed().as_secs(); debug!("File downloaded successfully to {}, took:{}", file_path.display(), format_elapsed_time(elapsed)); Ok(file_path.to_path_buf()) @@ -297,7 +304,7 @@ pub async fn get_remote_content_as_stream( let stream_reader = StreamReader::new( response.bytes_stream().map_err(std::io::Error::other), ); - let mut buf_reader = BufReader::new(stream_reader); + let mut buf_reader = async_file_reader(stream_reader); let peek = buf_reader.fill_buf().await?; if peek.len() >= 2 { diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index 5f7b88406..4c1a20cd0 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "frontend" -version = "3.2.17" +version = "3.2.18" edition = "2021" [dependencies] -shared = { version = "3.2.17", path = "../shared" } +shared = { version = "3.2.18", path = "../shared" } chrono = "0" yew = "0.21" yew-router = "0.18" diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 85658c486..5dacf7d4a 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shared" -version = "3.2.17" +version = "3.2.18" edition = "2021" [dependencies] diff --git a/shared/src/utils/string_utils.rs b/shared/src/utils/string_utils.rs index 08d18ffa4..f71d15e6c 100644 --- a/shared/src/utils/string_utils.rs +++ b/shared/src/utils/string_utils.rs @@ -131,11 +131,34 @@ pub fn longest<'a>(a: &'a str, b: &'a str) -> &'a str { if a.len() >= b.len() { a } else { b } } +// ------------------------------------------------------------ +// Generic string concatenation macro with optional capacity hint +// Usage: +// let s = concat_string!("/", user, "/", pass, "/", id); +// let s = concat_string!(cap = 128; prefix, "/", value); +// The macro writes all arguments using Display into a preallocated String +// to minimize temporary allocations and copies. +// ------------------------------------------------------------ +#[macro_export] +macro_rules! concat_string { + (cap = $cap:expr; $($arg:expr),* $(,)?) => {{ + let mut __s = ::std::string::String::with_capacity($cap); + $( let _ = ::std::fmt::Write::write_fmt(&mut __s, format_args!("{}", $arg)); )* + __s + }}; + ( $($arg:expr),* $(,)?) => {{ + let mut __s = ::std::string::String::new(); + $( let _ = ::std::fmt::Write::write_fmt(&mut __s, format_args!("{}", $arg)); )* + __s + }}; +} + #[cfg(test)] mod test { use std::collections::HashSet; use crate::utils::Capitalize; use super::generate_random_string; + use crate as shared; // allow path-based macro call in tests #[test] fn test_generate_random_string() { @@ -151,4 +174,20 @@ mod test { assert_eq!("hELLO".capitalize(), "Hello"); } + #[test] + fn test_concat_string_basic() { + let a = "hello"; + let b = String::from("world"); + let n = 42; + let s = shared::concat_string!(a, " ", b, " ", n); + assert_eq!(s, "hello world 42"); + } + + #[test] + fn test_concat_string_with_cap() { + let part = "abc"; + let s = shared::concat_string!(cap = 16; part, "/", 123); + assert_eq!(s, "abc/123"); + } + } From a522d2863bec4476487d7380df72bd5809848ab6 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 19:50:31 +0100 Subject: [PATCH 04/11] write operations optimized for less memory usage --- backend/src/api/endpoints/download_api.rs | 4 ++-- .../api/model/streams/persist_pipe_stream.rs | 1 + backend/src/api/scheduler.rs | 1 - .../src/processing/processor/xtream_series.rs | 2 +- backend/src/repository/indexed_document.rs | 7 ++---- backend/src/repository/m3u_repository.rs | 4 ++-- backend/src/repository/strm_repository.rs | 4 ++-- .../repository/xtream_playlist_iterator.rs | 8 +++++++ .../compressed_file_reader_async.rs | 24 ++++++++----------- backend/src/utils/file/file_utils.rs | 10 ++++---- backend/src/utils/network/request.rs | 4 ++-- 11 files changed, 35 insertions(+), 34 deletions(-) diff --git a/backend/src/api/endpoints/download_api.rs b/backend/src/api/endpoints/download_api.rs index 618d3cc20..918d30812 100644 --- a/backend/src/api/endpoints/download_api.rs +++ b/backend/src/api/endpoints/download_api.rs @@ -1,7 +1,7 @@ use crate::api::model::AppState; use crate::api::model::{DownloadQueue, FileDownload, FileDownloadRequest}; use crate::model::{AppConfig, VideoDownloadConfig}; -use crate::utils::{async_file_writer, request, WRITER_BUFFER_SIZE}; +use crate::utils::{async_file_writer, request, IO_BUFFER_SIZE}; use tokio::sync::RwLock; use futures::stream::TryStreamExt; use log::info; @@ -37,7 +37,7 @@ async fn download_file(active: Arc>>, client: &reqwe match buf_writer.write_all(&chunk).await { Ok(()) => { write_counter += chunk.len(); - if write_counter >= WRITER_BUFFER_SIZE { + if write_counter >= IO_BUFFER_SIZE { buf_writer.flush().await.map_err(|err| err.to_string())?; write_counter = 0; } diff --git a/backend/src/api/model/streams/persist_pipe_stream.rs b/backend/src/api/model/streams/persist_pipe_stream.rs index 984ec1001..d568bc036 100644 --- a/backend/src/api/model/streams/persist_pipe_stream.rs +++ b/backend/src/api/model/streams/persist_pipe_stream.rs @@ -40,6 +40,7 @@ where S: tokio_stream::Stream> + Send + Unpin if write_counter > FLUSH_INTERVAL { write_counter = 0; if let Err(err) = writer.flush().await { + writer_active = false; write_err = Some(StreamError::StdIo(format!("Failed periodic flush of tee_stream writer {err}"))); } } diff --git a/backend/src/api/scheduler.rs b/backend/src/api/scheduler.rs index 7b511d092..bdf97f9bf 100644 --- a/backend/src/api/scheduler.rs +++ b/backend/src/api/scheduler.rs @@ -55,7 +55,6 @@ async fn start_scheduler(client: reqwest::Client, expression: &str, app_state: A loop { let mut upcoming = schedule.upcoming(offset).take(1); if let Some(datetime) = upcoming.next() { - let client = client.clone(); tokio::select! { () = tokio::time::sleep_until(tokio::time::Instant::from(datetime_to_instant(datetime))) => { let app_config = Arc::clone(&app_state.app_config); diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 79c3b1aef..ee037cb8f 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -203,7 +203,7 @@ async fn process_series_info( } write_counter +=1; // periodic flush to bound BufWriter memory - if write_counter <= FLUSH_INTERVAL { + if write_counter >= FLUSH_INTERVAL { write_counter = 0; if let Err(err) = wal_writer.flush() { errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); diff --git a/backend/src/repository/indexed_document.rs b/backend/src/repository/indexed_document.rs index b0ab04eee..9323e12d7 100644 --- a/backend/src/repository/indexed_document.rs +++ b/backend/src/repository/indexed_document.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use tempfile::NamedTempFile; use shared::error::{str_to_io_error, to_io_error}; use crate::utils; -use crate::utils::{bincode_deserialize, bincode_serialize, WRITER_BUFFER_SIZE}; +use crate::utils::{bincode_deserialize, bincode_serialize, IO_BUFFER_SIZE}; const BLOCK_SIZE: usize = 4096; const LEN_SIZE: usize = 4; @@ -280,9 +280,6 @@ where if let Some(offset) = self.index_tree.query(doc_id) { self.main_file.seek(SeekFrom::Start(u64::from(*offset)))?; let buf_size = IndexedDocument::read_content_size(&mut self.main_file)?; - if self.buffer.capacity() < buf_size { - self.buffer.reserve(buf_size - self.buffer.capacity()); - } self.buffer.resize(buf_size, 0u8); self.main_file.read_exact(&mut self.buffer[..buf_size])?; if let Ok(item) = bincode_deserialize::(&self.buffer[..buf_size]) { @@ -514,7 +511,7 @@ where gc_writer.write_all(&size_bytes)?; gc_writer.write_all(&buffer[0..buf_size])?; write_counter += buf_size; - if write_counter >= WRITER_BUFFER_SIZE { + if write_counter >= IO_BUFFER_SIZE { write_counter = 0; gc_writer.flush()?; } diff --git a/backend/src/repository/m3u_repository.rs b/backend/src/repository/m3u_repository.rs index 97c68384a..a4c5f7a5d 100644 --- a/backend/src/repository/m3u_repository.rs +++ b/backend/src/repository/m3u_repository.rs @@ -17,7 +17,7 @@ use log::error; use tokio::fs; use tokio::io::{AsyncWriteExt}; use tokio::task; -use crate::utils::{async_file_writer, WRITER_BUFFER_SIZE}; +use crate::utils::{async_file_writer, IO_BUFFER_SIZE}; macro_rules! cant_write_result { ($path:expr, $err:expr) => { @@ -66,7 +66,7 @@ async fn persist_m3u_playlist_as_text( await_playlist_write!(writer.write_all(bytes), "Failed to write entry to {} - {}", m3u_filename.display()); await_playlist_write!(writer.write_all(b"\n"), "Failed to write newline to {} - {}", m3u_filename.display()); write_counter += bytes.len() + 1; - if write_counter >= WRITER_BUFFER_SIZE { + if write_counter >= IO_BUFFER_SIZE { await_playlist_write!(writer.flush(), "Failed to flush {} - {}", m3u_filename.display()); write_counter = 0; } diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index f3628b215..599ffe759 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -8,7 +8,7 @@ use crate::repository::storage::{ensure_target_storage_path, get_input_storage_p use crate::repository::storage_const; use crate::repository::xtream_repository::{xtream_get_record_file_path, InputVodInfoRecord}; use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; -use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, FileReadGuard, WRITER_BUFFER_SIZE}; +use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, FileReadGuard, IO_BUFFER_SIZE}; use chrono::Datelike; use filetime::{set_file_times, FileTime}; use log::{error, trace}; @@ -947,7 +947,7 @@ async fn write_strm_index_file( .write(new_line) .await .map_err(|err| format!("Failed to write strm index entry: {err}"))?; - if write_counter >= WRITER_BUFFER_SIZE { + if write_counter >= IO_BUFFER_SIZE { write_counter = 0; writer.flush().await.map_err(|err| format!("Failed to flush: {err}"))?; } diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index 8b64d3714..ddf703e7e 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -54,6 +54,14 @@ impl XtreamPlaylistIterator { let filter_ids: Option> = filter.as_ref().map(|set| { set.iter().filter_map(|s| s.parse::().ok()).collect() }); + let filter_ids: Option> = filter.as_ref().map(|set| { + set.iter().filter_map(|s| { + s.parse::().map_err(|e| { + error!("Failed to parse bouquet filter id '{}': {}", s, e); + e + }).ok() + }).collect() + }); Ok(Self { reader, diff --git a/backend/src/utils/compression/compressed_file_reader_async.rs b/backend/src/utils/compression/compressed_file_reader_async.rs index cc6d7e30f..a62c92b45 100644 --- a/backend/src/utils/compression/compressed_file_reader_async.rs +++ b/backend/src/utils/compression/compressed_file_reader_async.rs @@ -1,16 +1,16 @@ +use crate::utils::async_file_reader; +use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; +use async_compression::tokio::bufread::{GzipDecoder, ZlibDecoder}; use std::path::Path; use std::pin::Pin; use std::task::{Context, Poll}; use tokio::fs::File; use tokio::io::{ - self, AsyncRead, BufReader, AsyncSeekExt, AsyncReadExt, ReadBuf, + self, AsyncRead, AsyncReadExt, AsyncSeekExt, ReadBuf, }; -use async_compression::tokio::bufread::{GzipDecoder, ZlibDecoder}; -use crate::utils::async_file_reader; -use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; pub struct CompressedFileReaderAsync { - reader: BufReader>, + reader: Box, } impl CompressedFileReaderAsync { @@ -22,17 +22,13 @@ impl CompressedFileReaderAsync { buffered_file.read_exact(&mut header).await?; buffered_file.seek(io::SeekFrom::Start(0)).await?; - let reader: Box = if is_gzip(&header) { - Box::new(GzipDecoder::new(buffered_file)) + if is_gzip(&header) { + Ok(Self { reader: Box::new(GzipDecoder::new(buffered_file)) }) } else if is_deflate(&header) { - Box::new(ZlibDecoder::new(buffered_file)) + Ok(Self { reader: Box::new(ZlibDecoder::new(buffered_file)) }) } else { - Box::new(buffered_file) - }; - - Ok(Self { - reader: async_file_reader(reader), - }) + Ok(Self { reader: Box::new(buffered_file) }) + } } } diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index 1e719ded8..698e8a41a 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -11,34 +11,34 @@ use log::{debug, error}; use path_clean::PathClean; use tokio::fs as tokio_fs; -pub const WRITER_BUFFER_SIZE: usize = 256*1024; // 256kb +pub const IO_BUFFER_SIZE: usize = 256*1024; // 256kb pub fn file_writer(w: W) -> std::io::BufWriter where W: std::io::Write, { - std::io::BufWriter::with_capacity(WRITER_BUFFER_SIZE, w) + std::io::BufWriter::with_capacity(IO_BUFFER_SIZE, w) } pub fn file_reader(r: R) -> std::io::BufReader where R: std::io::Read, { - std::io::BufReader::with_capacity(WRITER_BUFFER_SIZE, r) + std::io::BufReader::with_capacity(IO_BUFFER_SIZE, r) } pub fn async_file_writer(w: W) -> tokio::io::BufWriter where W: tokio::io::AsyncWrite, { - tokio::io::BufWriter::with_capacity(WRITER_BUFFER_SIZE, w) + tokio::io::BufWriter::with_capacity(IO_BUFFER_SIZE, w) } pub fn async_file_reader(r: R) -> tokio::io::BufReader where R: tokio::io::AsyncRead, { - tokio::io::BufReader::with_capacity(WRITER_BUFFER_SIZE, r) + tokio::io::BufReader::with_capacity(IO_BUFFER_SIZE, r) } diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 3986c0eac..b184291c8 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -20,7 +20,7 @@ use crate::model::{ConfigInput}; use crate::repository::storage::{get_input_storage_path}; use crate::repository::storage_const; use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; -use crate::utils::{async_file_reader, async_file_writer, debug_if_enabled, WRITER_BUFFER_SIZE}; +use crate::utils::{async_file_reader, async_file_writer, debug_if_enabled, IO_BUFFER_SIZE}; use shared::utils::{filter_request_header, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP}; use crate::utils::{get_file_path, persist_file}; @@ -257,7 +257,7 @@ async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInpu Ok(bytes) => { write_counter += bytes.len(); writer.write_all(&bytes).await?; - if write_counter >= WRITER_BUFFER_SIZE { + if write_counter >= IO_BUFFER_SIZE { writer.flush().await?; write_counter = 0; } From 09452d7b85d015df4e6c520388ebe8226deb7c3b Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 19:51:39 +0100 Subject: [PATCH 05/11] write operations optimized for less memory usage --- backend/src/repository/xtream_playlist_iterator.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index ddf703e7e..9d0b1619f 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -51,13 +51,10 @@ impl XtreamPlaylistIterator { let filter = user_get_bouquet_filter(&config, &user.username, category_id, TargetType::Xtream, cluster).await; // Parse bouquet filter (strings) once into u32 set to minimize per-item allocations - let filter_ids: Option> = filter.as_ref().map(|set| { - set.iter().filter_map(|s| s.parse::().ok()).collect() - }); let filter_ids: Option> = filter.as_ref().map(|set| { set.iter().filter_map(|s| { s.parse::().map_err(|e| { - error!("Failed to parse bouquet filter id '{}': {}", s, e); + error!("Failed to parse bouquet filter id '{s}': {e}"); e }).ok() }).collect() From 80660312e4f67b560c91b931fabe41520a2458cd Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 20:02:20 +0100 Subject: [PATCH 06/11] write operations optimized for less memory usage --- backend/src/api/model/streams/persist_pipe_stream.rs | 7 +++---- backend/src/model/xmltv.rs | 2 +- backend/src/processing/processor/xtream_series.rs | 10 +++++----- backend/src/processing/processor/xtream_vod.rs | 7 +++---- backend/src/repository/strm_repository.rs | 2 +- backend/src/utils/file/file_utils.rs | 2 +- backend/src/utils/network/request.rs | 2 ++ 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/backend/src/api/model/streams/persist_pipe_stream.rs b/backend/src/api/model/streams/persist_pipe_stream.rs index d568bc036..42642a913 100644 --- a/backend/src/api/model/streams/persist_pipe_stream.rs +++ b/backend/src/api/model/streams/persist_pipe_stream.rs @@ -6,8 +6,7 @@ use std::sync::Arc; use tokio::io::AsyncWriteExt; use tokio_stream::{StreamExt}; use tokio_stream::wrappers::ReceiverStream; - -const FLUSH_INTERVAL: usize = 50; +use crate::utils::IO_BUFFER_SIZE; pub fn tee_stream( mut stream: S, @@ -36,8 +35,8 @@ where S: tokio_stream::Stream> + Send + Unpin writer_active = false; write_err = Some(StreamError::StdIo(e.to_string())); } else { - write_counter += 1; - if write_counter > FLUSH_INTERVAL { + write_counter += bytes.len(); + if write_counter >= IO_BUFFER_SIZE { write_counter = 0; if let Err(err) = writer.flush().await { writer_active = false; diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index 5ec19fc6c..a47652c6d 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -126,7 +126,7 @@ impl Epg { } } write_counter += 1; - if write_counter > 50 { + if write_counter >= 50 { writer.get_mut().flush().await?; // flush underlying writer write_counter = 0; } diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index ee037cb8f..d4fdd7ec7 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -19,7 +19,7 @@ use log::{error, info, log_enabled, warn, Level}; use crate::model::{XtreamSeriesEpisode, XtreamSeriesInfoEpisode}; use crate::utils; use crate::processing::processor::xtream::normalize_json_content; -use crate::utils::bincode_serialize; +use crate::utils::{bincode_serialize, IO_BUFFER_SIZE}; create_resolve_options_function_for_xtream_target!(series); @@ -93,10 +93,10 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}")))); processed_info_ids.insert(provider_id, ts); content_updated = true; - write_counter += 1; + write_counter += normalized_str.len(); // periodic flush to bound BufWriter memory - if write_counter >= FLUSH_INTERVAL { + if write_counter >= IO_BUFFER_SIZE { write_counter = 0; if let Err(err) = content_writer.flush() { errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); @@ -121,9 +121,9 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, // record_wal contains provider_id and timestamp if content_updated { handle_error!(content_writer.flush(), - |err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}")))); + |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}")))); handle_error!(record_writer.flush(), - |err| errors.push(notify_err!(format!("Failed to resolve vod tmdb, could not write to wal file {err}")))); + |err| errors.push(notify_err!(format!("Failed to resolve series tmdb, could not write to wal file {err}")))); handle_error!(content_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}")))); handle_error!(record_writer.get_ref().sync_all(), |err| errors.push(notify_err!(format!("Failed to sync series info record to wal file {err}")))); drop(content_writer); diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index a9189f72f..48500781e 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -15,6 +15,7 @@ use std::time::Instant; use log::{info, log_enabled, Level}; use crate::utils; use crate::processing::processor::xtream::normalize_json_content; +use crate::utils::IO_BUFFER_SIZE; create_resolve_options_function_for_xtream_target!(vod); @@ -63,8 +64,6 @@ fn should_update_vod_info(pli: &mut PlaylistItem, processed_provider_ids: &HashM should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_VOD_INFO_ADDED) } -const FLUSH_INTERVAL: usize = 50; - pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>) { let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl); if !resolve_movies { return; } @@ -108,9 +107,9 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Clie |err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}")))); processed_info_ids.insert(provider_id, ts); content_updated = true; - write_counter += 1; + write_counter += normalized_str.len(); // periodic flush to bound BufWriter memory - if write_counter >= FLUSH_INTERVAL { + if write_counter >= IO_BUFFER_SIZE { write_counter = 0; if let Err(err) = content_writer.flush() { errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 599ffe759..0cfdeb873 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -944,7 +944,7 @@ async fn write_strm_index_file( .await .map_err(|err| format!("Failed to write strm index entry: {err}"))?; writer - .write(new_line) + .write_all(new_line) .await .map_err(|err| format!("Failed to write strm index entry: {err}"))?; if write_counter >= IO_BUFFER_SIZE { diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index 698e8a41a..ba677119e 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; use std::{env, fs}; -use std::io::Read as IORead; +use std::io::Read; use shared::error::str_to_io_error; use crate::utils::debug_if_enabled; use shared::utils::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, MAPPING_FILE, SOURCE_FILE, USER_FILE}; diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index b184291c8..9f8baaa82 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -263,6 +263,8 @@ async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInpu } } Err(err) => { + let _ = writer.flush().await; + let _ = writer.shutdown().await; return Err(str_to_io_error(&format!("Failed to read chunk: {err}"))); } } From c22a3df93b89e2b1af4ee3122a258f9eb972e2e8 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 11 Dec 2025 20:18:31 +0100 Subject: [PATCH 07/11] write operations optimized for less memory usage --- .../src/processing/processor/xtream_series.rs | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index d4fdd7ec7..cfa7f2da4 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -31,26 +31,25 @@ fn write_series_episode_record_to_wal_file( writer: &mut BufWriter<&File>, provider_id: u32, episode: &XtreamSeriesInfoEpisode, -) -> std::io::Result<()> { +) -> std::io::Result { let series_episode = XtreamSeriesEpisode::from(episode); if let Ok(content_bytes) = bincode_serialize(&series_episode) { writer.write_all(&provider_id.to_le_bytes())?; - if let Ok(len) = u32::try_from(content_bytes.len()) { + let content_len = content_bytes.len(); + if let Ok(len) = u32::try_from(content_len) { writer.write_all(&len.to_le_bytes())?; writer.write_all(&content_bytes)?; - } else { - error!("Cant write to WAL file, content length exceeds u32"); + return Ok(content_len + 4usize) } + error!("Cant write to WAL file, content length exceeds u32"); } - Ok(()) + Ok(0) } fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap) -> (bool, u32, u64) { should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_SERIES_INFO_LAST_MODIFIED) } -const FLUSH_INTERVAL: usize = 50; - async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { let mut processed_info_ids: HashMap = read_processed_series_info_ids(cfg, errors, fpl).await; @@ -198,15 +197,18 @@ async fn process_series_info( Ok(Some(mut series)) => { for (episode, pli_episode) in &mut series { let Some(provider_id) = &pli_episode.header.get_provider_id() else { continue; }; - handle_error!(write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, episode), - |err| errors.push(info_err!(format!("Failed to write to series episode wal file: {err}")))); - } - write_counter +=1; - // periodic flush to bound BufWriter memory - if write_counter >= FLUSH_INTERVAL { - write_counter = 0; - if let Err(err) = wal_writer.flush() { - errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); + match write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, episode) { + Ok(written_bytes) => { + write_counter += written_bytes; + // periodic flush to bound BufWriter memory + if write_counter >= IO_BUFFER_SIZE { + write_counter = 0; + if let Err(err) = wal_writer.flush() { + errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}"))); + } + } + } + Err(err) => {errors.push(info_err!(format!("Failed to write to series episode wal file: {err}"))) } } } group_series.extend(series.into_iter().map(|(_, pli)| pli)); From ee0441877d4cf183827bbdde9df18afffb64a7ec Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 12 Dec 2025 14:35:00 +0100 Subject: [PATCH 08/11] xtream and m3u parsing with streaming --- backend/Cargo.toml | 2 +- backend/src/api/api_utils.rs | 13 +- backend/src/api/endpoints/hls_api.rs | 6 +- backend/src/api/endpoints/v1_api.rs | 5 +- backend/src/api/endpoints/v1_api_config.rs | 6 +- backend/src/api/main_api.rs | 2 + backend/src/api/model/app_state.rs | 15 +- backend/src/api/model/streams/mod.rs | 5 +- .../api/model/streams/persist_pipe_stream.rs | 67 +++++- backend/src/model/xmltv.rs | 23 ++- backend/src/processing/parser/m3u.rs | 49 ++--- backend/src/processing/parser/xtream.rs | 52 +++-- backend/src/processing/processor/playlist.rs | 6 +- .../src/processing/processor/xtream_vod.rs | 2 + backend/src/utils/file/csv_input_reader.rs | 2 +- backend/src/utils/geoip.rs | 1 - backend/src/utils/json_utils.rs | 15 ++ backend/src/utils/network/m3u.rs | 6 +- backend/src/utils/network/request.rs | 193 +++++++++++++++--- backend/src/utils/network/xtream.rs | 8 +- shared/src/utils/serde_utils.rs | 63 +++--- 21 files changed, 388 insertions(+), 153 deletions(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index cbba13ba1..ebb758e1b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -47,7 +47,7 @@ tokio = { version = "1.48", features = ["rt-multi-thread", "parking_lot", "fs"] #console-subscriber = "0" #tracing = "0.1" #tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } -tokio-util = { version = "0.7", features = ["io"] } +tokio-util = { version = "0.7", features = ["io", "io-util"] } tempfile = "3.23" ruzstd = "0.8" filetime = "0.2" diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index 88615182d..0d2dd0c8a 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -479,13 +479,7 @@ async fn create_stream_response_details( | ProviderStreamState::GracePeriod(_provider_name, request_url) => { let parsed_url = Url::parse(&request_url); let ((stream, stream_info), reconnect_flag) = if let Ok(url) = parsed_url { - let disabled_headers = app_state - .app_config - .config - .load() - .reverse_proxy - .as_ref() - .and_then(|r| r.disabled_header.clone()); + let disabled_headers = app_state.get_disabled_headers(); let provider_stream_factory_options = ProviderStreamFactoryOptions::new( fingerprint.addr, item_type, @@ -1135,10 +1129,7 @@ async fn fetch_resource_with_retry( .reverse_proxy .as_ref() .map_or_else(ResourceRetryConfig::get_default_retry_values, |rp| rp.resource_retry.get_retry_values()); - let disabled_headers = config - .reverse_proxy - .as_ref() - .and_then(|r| r.disabled_header.clone()); + let disabled_headers = app_state.get_disabled_headers(); for attempt in 0..max_attempts { let client = request::get_client_request( &app_state.http_client.load(), diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 57f0fb53b..1f8a890b8 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -111,11 +111,7 @@ pub(in crate::api) async fn handle_hls_stream_request( // Don't forward Range on playlist fetch; segments use original headers in provider path let filter_header: HeaderFilter = Some(Box::new(|name: &str| !name.eq_ignore_ascii_case("range"))); let forwarded = get_headers_from_request(req_headers, &filter_header); - let config = app_state.app_config.config.load(); - let disabled_headers = config - .reverse_proxy - .as_ref() - .and_then(|r| r.disabled_header.clone()); + let disabled_headers = app_state.get_disabled_headers(); let headers = request::get_request_headers(None, Some(&forwarded), disabled_headers.as_ref()); let input_source = InputSource::from(input).with_url(request_url); match request::download_text_content( diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index 39ba81507..558932a34 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -89,10 +89,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State { let reader = Cursor::new(content); diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 8a4a6c64f..7834e430a 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -118,11 +118,7 @@ async fn config_batch_content( // The url is changed at this point, we need the raw url for the batch file if let Some(batch_url) = config_input.t_batch_url.as_ref() { let input_source = InputSource::from(&*config_input).with_url(batch_url.to_owned()); - let config = app_state.app_config.config.load(); - let disabled_headers = config - .reverse_proxy - .as_ref() - .and_then(|r| r.disabled_header.clone()); + let disabled_headers = app_state.get_disabled_headers(); return match download_text_content(&app_state.http_client.load(), disabled_headers.as_ref(), &input_source, None, None).await { Ok((content, _path)) => { // Return CSV with explicit content-type diff --git a/backend/src/api/main_api.rs b/backend/src/api/main_api.rs index 6d2948bbd..14d6bf901 100644 --- a/backend/src/api/main_api.rs +++ b/backend/src/api/main_api.rs @@ -31,6 +31,7 @@ use std::sync::atomic::AtomicI8; use std::sync::Arc; use tokio_util::sync::CancellationToken; use tower_governor::key_extractor::SmartIpKeyExtractor; +use tower_http::services::ServeDir; use crate::api::sys_usage::exec_system_usage; use crate::repository::storage::get_geoip_path; use crate::utils::{exec_file_lock_prune, GeoIp}; @@ -315,6 +316,7 @@ pub async fn start_server( // Web Server let mut router = axum::Router::new() .route("/healthcheck", axum::routing::get(healthcheck)) + .nest_service("/.well-known", ServeDir::new(web_dir_path.join("static/.well-known"))) .merge(ws_api_register( web_auth_enabled, web_ui_path.as_str(), diff --git a/backend/src/api/model/app_state.rs b/backend/src/api/model/app_state.rs index 146ed1c8f..d27b894f6 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/src/api/model/app_state.rs @@ -2,10 +2,7 @@ use crate::api::config_watch::exec_config_watch; use crate::api::model::{ActiveProviderManager, ConnectionManager, EventManager, PlaylistStorage, PlaylistStorageState, SharedStreamManager}; use crate::api::model::{ActiveUserManager, DownloadQueue}; use crate::api::scheduler::exec_scheduler; -use crate::model::{ - AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets, - ScheduleConfig, SourcesConfig, -}; +use crate::model::{AppConfig, Config, ConfigTarget, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets, ReverseProxyDisabledHeaderConfig, ScheduleConfig, SourcesConfig}; use crate::repository::playlist_repository::load_target_into_memory_cache; use crate::tools::lru_cache::LRUResourceCache; use crate::utils::request::create_client; @@ -444,6 +441,16 @@ impl AppState { pub async fn cache_playlist(&self, target_name: &str, playlist: PlaylistStorage) { self.playlists.cache_playlist(target_name, playlist).await; } + + pub fn get_disabled_headers(&self) -> Option { + self + .app_config + .config + .load() + .reverse_proxy + .as_ref() + .and_then(|r| r.disabled_header.clone()) + } } fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool { diff --git a/backend/src/api/model/streams/mod.rs b/backend/src/api/model/streams/mod.rs index d7a686da4..42aad1a9c 100644 --- a/backend/src/api/model/streams/mod.rs +++ b/backend/src/api/model/streams/mod.rs @@ -5,17 +5,18 @@ mod custom_video_stream; mod transport_stream_buffer; // mod chunked_buffer; mod provider_stream; -mod persist_pipe_stream; mod provider_stream_factory; mod shared_stream_manager; mod active_client_stream; mod throttled_stream; +pub mod persist_pipe_stream; + pub(in crate) use self::transport_stream_buffer::*; pub(in crate::api) use self::provider_stream::*; -pub(in crate::api) use self::persist_pipe_stream::*; pub(in crate::api) use self::provider_stream_factory::*; pub(in crate::api) use self::shared_stream_manager::*; pub(in crate::api) use self::active_client_stream::*; pub(in crate::api) use self::throttled_stream::*; pub(in crate::api) use self::timed_client_stream::*; pub(in crate::api) use self::custom_video_stream::*; +pub use self::persist_pipe_stream::*; diff --git a/backend/src/api/model/streams/persist_pipe_stream.rs b/backend/src/api/model/streams/persist_pipe_stream.rs index 42642a913..32a8c2081 100644 --- a/backend/src/api/model/streams/persist_pipe_stream.rs +++ b/backend/src/api/model/streams/persist_pipe_stream.rs @@ -1,12 +1,13 @@ -use std::path::Path; -use crate::api::model::StreamError; +use crate::utils::request::DynReader; +use crate::utils::{async_file_writer, IO_BUFFER_SIZE}; use bytes::Bytes; use log::{debug, error}; +use std::path::{Path,}; use std::sync::Arc; -use tokio::io::AsyncWriteExt; -use tokio_stream::{StreamExt}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_stream::wrappers::ReceiverStream; -use crate::utils::IO_BUFFER_SIZE; +use tokio_stream::StreamExt; +use crate::api::model::StreamError; pub fn tee_stream( mut stream: S, @@ -14,8 +15,9 @@ pub fn tee_stream( file_path: &Path, callback: Arc, ) -> ReceiverStream> -where S: tokio_stream::Stream> + Send + Unpin + 'static, - W: tokio::io::AsyncWrite + Send + Unpin + 'static, +where + S: tokio_stream::Stream> + Send + Unpin + 'static, + W: tokio::io::AsyncWrite + Send + Unpin + 'static, { let (tx, rx) = tokio::sync::mpsc::channel::>(32); let resource_path = file_path.to_owned(); @@ -77,3 +79,54 @@ where S: tokio_stream::Stream> + Send + Unpin ReceiverStream::new(rx) } + +pub async fn tee_dyn_reader( + reader: DynReader, + persist_path: &Path, + callback: Option>, +) -> DynReader { + let file = match tokio::fs::File::create(persist_path).await { + Ok(f) => f, + Err(err) => { + error!("Cant open file to write: {}, {err}", persist_path.display()); + return reader; + } + }; + + let (mut tx, rx) = tokio::io::duplex(IO_BUFFER_SIZE); + let mut writer = async_file_writer(file); + let reader_arc = reader; + + tokio::spawn(async move { + let mut total_bytes = 0usize; + let mut buf = [0u8; 8192]; + + let mut reader = reader_arc; + + loop { + let n = match reader.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + + total_bytes += n; + + if tx.write_all(&buf[..n]).await.is_err() { + break; + } + + if writer.write_all(&buf[..n]).await.is_err() { + break; + } + } + + let _ = writer.flush().await; + let _ = tx.shutdown().await; + + if let Some(cb) = callback { + cb(total_bytes); + } + }); + + Box::pin(rx) as DynReader +} \ No newline at end of file diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index a47652c6d..138bceb6b 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -12,6 +12,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use url::Url; use shared::utils::sanitize_sensitive_info; use crate::api::model::AppState; +use crate::model::{InputSource}; use crate::utils::async_file_reader; use crate::utils::request::{get_remote_content_as_stream}; @@ -211,11 +212,23 @@ pub async fn parse_xmltv_for_web_ui_from_file(path: &Path) -> Result, url: &str) -> Result { if let Ok(request_url) = Url::parse(url) { let client = app_state.http_client.load(); - match get_remote_content_as_stream( - client.as_ref(), - &request_url, - InputFetchMethod::GET, - None, + let input_source: InputSource = InputSource { + name: String::from("xmltv"), + url: request_url.to_string(), + username: None, + password: None, + method: InputFetchMethod::GET, + headers: HashMap::default(), + }; + + let disabled_headers = app_state.get_disabled_headers(); + + match get_remote_content_as_stream( + &client, + &input_source, + None, + &request_url, + disabled_headers.as_ref(), ).await { Ok((stream, _url)) => { parse_xmltv_for_web_ui(stream).await diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 5f3bcf4a4..c910d0efd 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -2,7 +2,8 @@ use crate::model::{Config, ConfigInput}; use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster, DEFAULT_VIDEO_EXTENSIONS}; use shared::utils::extract_id_from_url; use std::borrow::BorrowMut; - +use tokio::io::AsyncBufReadExt; +use crate::utils::request::DynReader; // other implementations like calculating text_distance on all titles took too much time // we keep it now as simple as possible and less memory intensive. @@ -87,9 +88,9 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { } } -fn create_empty_playlistitem_header(input_name: &str, url: &str) -> PlaylistItemHeader { +fn create_empty_playlistitem_header(input_name: &str, url: String) -> PlaylistItemHeader { PlaylistItemHeader { - url: url.to_owned(), + url, category_id: 0, input_name: input_name.to_string(), ..Default::default() @@ -107,7 +108,15 @@ macro_rules! process_header_fields { }; } -fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: &str) -> PlaylistItemHeader { +fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: String) -> PlaylistItemHeader { + let url_id = extract_id_from_url(&url); + let url_types = if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { + // TODO find Series based on group or configured names + Some((XtreamCluster::Video, PlaylistItemType::Video)) + } else { + None + }; + let mut plih = create_empty_playlistitem_header(input_name, url); let mut it = content.chars(); let mut stack = String::with_capacity(64); @@ -157,7 +166,7 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: plih.epg_channel_id = None; if let Some(pid) = provider_id { plih.id = pid; - } else if let Some(chanid) = extract_id_from_url(url) { + } else if let Some(chanid) = url_id { plih.id = chanid; } } else { @@ -167,11 +176,9 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: } } } - - if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { - // TODO find Series based on group or configured names - plih.xtream_cluster = XtreamCluster::Video; - plih.item_type = PlaylistItemType::Video; + if let Some((url_cluster, url_item_type)) = url_types { + plih.xtream_cluster = url_cluster; + plih.item_type = url_item_type; } { @@ -189,10 +196,7 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url: plih } -pub fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F) -where - I: Iterator, -{ +pub async fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader, mut visit: F) { let mut header: Option = None; let mut group: Option = None; let input_name = input.name.as_str(); @@ -203,9 +207,10 @@ where }, None => DEFAULT_VIDEO_EXTENSIONS.to_vec() }; - for line in lines { + let mut lines = tokio::io::BufReader::new(lines).lines(); + while let Ok(Some(line)) = lines.next_line().await { if line.starts_with("#EXTINF") { - header = Some(String::from(line)); + header = Some(line); continue; } if line.starts_with("#EXTGRP") { @@ -233,9 +238,7 @@ where } } -pub fn parse_m3u<'a, I>(cfg: &Config, input: &ConfigInput, lines: I) -> Vec -where - I: Iterator, +pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> Vec { let mut sort_order: Vec> = vec![]; let mut sort_order_idx: usize = 0; @@ -258,7 +261,7 @@ where } } } - }); + }).await; let mut grp_id = 0; let result: Vec = sort_order.into_iter().filter_map(|channels| { // create a group based on the first playlist item @@ -285,7 +288,7 @@ mod test { let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney" , Seven"#; - let pli = process_header(input, &video_suffixes, line, url); + let pli = process_header(input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "abc-seven"); assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); @@ -300,7 +303,7 @@ mod test { let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 channel-id="abc-seven" tvg-id="abc-seven" tvg-logo="https://abc.nz/.images/seven.png" tvg-chno="7" group-title="Sydney", Seven"#; - let pli = process_header(input, &video_suffixes, line, url); + let pli = process_header(input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "abc-seven"); assert_eq!(pli.logo, "https://abc.nz/.images/seven.png"); @@ -315,7 +318,7 @@ mod test { let url = "http://hello.de/hello.ts"; let line = r#"#EXTINF:-1 tvg-id="abc-seven" xui-id="provider-123" group-title="Sydney", Seven"#; - let pli = process_header(input, &video_suffixes, line, url); + let pli = process_header(input, &video_suffixes, line, url.to_string()); assert_eq!(pli.title, "Seven"); assert_eq!(pli.id, "provider-123"); // Should use xui-id assert_eq!(pli.epg_channel_id, Some("abc-seven".to_string())); // Should preserve original tvg-id diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 9a63497f0..40e8074f9 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -7,23 +7,31 @@ use crate::utils::xtream::{get_xtream_stream_url_base}; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use tokio::task::spawn_blocking; +use crate::utils::request::DynReader; -fn map_to_xtream_category(categories: &Value) -> Result, TuliproxError> { - match serde_json::from_value::>(categories.to_owned()) { - Ok(xtream_categories) => Ok(xtream_categories), - Err(err) => { - create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process categories {}", &err) +async fn map_to_xtream_category(categories: DynReader) -> Result, TuliproxError> { + spawn_blocking(move || { + let reader = tokio_util::io::SyncIoBridge::new(categories); + match serde_json::from_reader::<_, Vec>(reader) { + Ok(xtream_categories) => Ok(xtream_categories), + Err(err) => { + create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process categories {}", &err) + } } - } + }).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream categories failed: {e}")))? } -fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: &Value) -> Result, TuliproxError> { - match serde_json::from_value::>(streams.to_owned()) { +async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader) -> Result, TuliproxError> { + spawn_blocking(move || { + let reader = tokio_util::io::SyncIoBridge::new(streams); + match serde_json::from_reader::<_, Vec>(reader) { Ok(stream_list) => Ok(stream_list), Err(err) => { - create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {:?}: {}", xtream_cluster, &err) + create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {xtream_cluster}: {err}", ) } } + }).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream streams failed: {e}")))? } fn create_xtream_series_episode_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Arc { @@ -119,18 +127,18 @@ pub fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &st } } -pub fn parse_xtream(input: &ConfigInput, +pub async fn parse_xtream(input: &ConfigInput, xtream_cluster: XtreamCluster, - categories: &Value, - streams: &Value) -> Result>, TuliproxError> { - match map_to_xtream_category(categories) { + categories: DynReader, + streams: DynReader) -> Result>, TuliproxError> { + match map_to_xtream_category(categories).await { Ok(xtream_categories) => { let input_name = input.name.clone(); let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); - match map_to_xtream_streams(xtream_cluster, streams) { + match map_to_xtream_streams(xtream_cluster, streams).await { Ok(mut xtream_streams) => { let mut group_map: HashMap = xtream_categories.into_iter().map(|category| @@ -204,7 +212,10 @@ pub fn parse_xtream(input: &ConfigInput, #[cfg(test)] mod tests { use std::fs; + use shared::model::XtreamCluster; use crate::model::XtreamSeriesInfo; + use crate::processing::parser::xtream::map_to_xtream_streams; + use crate::utils::async_file_reader; #[test] fn test_read_json_file_into_struct() { @@ -221,4 +232,17 @@ mod tests { } + #[tokio::test] + async fn test_read_json_stream_into_struct() -> std::io::Result<()> { + let reader = Box::pin(async_file_reader(tokio::fs::File::open("/tmp/vod_streams.json").await?)); + match map_to_xtream_streams(XtreamCluster::Video, reader).await { + Ok(_streams) => { + assert!(true); + }, + Err(err) => { + assert!(false, "Failed to parse json file: {err}"); + } + }; + Ok(()) + } } \ No newline at end of file diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 710bbf190..198a23647 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -24,7 +24,7 @@ use crate::processing::processor::trakt::process_trakt_categories_for_target; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::persist_playlist; -use crate::utils::debug_if_enabled; +use crate::utils::{debug_if_enabled, trace_if_enabled}; use crate::utils::StepMeasure; use deunicode::deunicode; use futures::StreamExt; @@ -134,7 +134,7 @@ fn exec_rename(pli: &mut PlaylistItem, rename: Option<&Vec>) { let value = get_field_value(result, r.field); let cap = r.pattern.replace_all(value.as_str(), &r.new_name); if log_enabled!(log::Level::Debug) && *value != cap { - debug_if_enabled!("Renamed {}={} to {}", &r.field, value, cap); + trace_if_enabled!("Renamed {}={value} to {cap}", &r.field); } let value = cap.into_owned(); set_field_value(result, r.field, value); @@ -153,7 +153,7 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt for r in renames { if matches!(r.field, ItemField::Group) { let cap = r.pattern.replace_all(&grp.title, &r.new_name); - debug_if_enabled!("Renamed group {} to {} for {}", &grp.title, cap, target.name); + trace_if_enabled!("Renamed group {} to {cap} for {}", &grp.title, target.name); grp.title = cap.into_owned(); } } diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 48500781e..26810540f 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -68,6 +68,8 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Clie let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl); if !resolve_movies { return; } + // TODO read existing WAL File and import it to avoid duplicate requests + // we cant write to the indexed-document directly because of the write lock and time-consuming operation. // All readers would be waiting for the lock and the app would be unresponsive. // We collect the content into a wal file and write it once we collected everything. diff --git a/backend/src/utils/file/csv_input_reader.rs b/backend/src/utils/file/csv_input_reader.rs index 17fd6faf6..0378d893b 100644 --- a/backend/src/utils/file/csv_input_reader.rs +++ b/backend/src/utils/file/csv_input_reader.rs @@ -190,7 +190,7 @@ pub fn get_csv_file_path(file_uri: &str) -> Result { mod tests { use crate::utils::file::csv_input_reader::csv_read_inputs_from_reader; use crate::utils::{file_reader, resolve_env_var}; - use std::io::{BufReader, Cursor}; + use std::io::{Cursor}; use shared::model::InputType; const M3U_BATCH: &str = r" diff --git a/backend/src/utils/geoip.rs b/backend/src/utils/geoip.rs index e5cadd7d1..07af9ddf8 100644 --- a/backend/src/utils/geoip.rs +++ b/backend/src/utils/geoip.rs @@ -98,7 +98,6 @@ mod test { use crate::utils::geoip::GeoIp; use std::fs::File; - use std::io::BufReader; use std::path::PathBuf; use crate::utils::file_reader; diff --git a/backend/src/utils/json_utils.rs b/backend/src/utils/json_utils.rs index 43888607a..99adfe0ac 100644 --- a/backend/src/utils/json_utils.rs +++ b/backend/src/utils/json_utils.rs @@ -54,3 +54,18 @@ where buf_writer.flush()?; buf_writer.into_inner()?.sync_all() } + +// pub async fn is_valid_json_file(path: &str) -> std::io::Result { +// if let Ok(file) = tokio::fs::File::open(path).await { +// let reader = async_file_reader(file); +// let stream = serde_json::Deserializer::from_reader(reader).into_iter::(); +// for item in stream { +// if item.is_err() { +// return Ok(false); +// } +// } +// Ok(true) +// } else { +// Ok(false) +// } +// } \ No newline at end of file diff --git a/backend/src/utils/network/m3u.rs b/backend/src/utils/network/m3u.rs index d640d177e..dd12c7f42 100644 --- a/backend/src/utils/network/m3u.rs +++ b/backend/src/utils/network/m3u.rs @@ -14,9 +14,9 @@ pub async fn get_m3u_playlist(client: &reqwest::Client, cfg: &Arc, input } }; let persist_file_path = prepare_file_path(input.persist.as_deref(), working_dir, ""); - match request::get_input_text_content(client, &input_source, working_dir, persist_file_path).await { - Ok(text) => { - (m3u::parse_m3u(cfg, input, text.lines()), vec![]) + match request::get_input_text_content_as_stream(client, &input_source, working_dir, persist_file_path).await { + Ok(reader) => { + (m3u::parse_m3u(cfg, input, reader).await, vec![]) } Err(err) => (vec![], vec![err]) } diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 9f8baaa82..6e3e14627 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -1,28 +1,30 @@ -use std::collections::{HashMap, HashSet}; -use std::io::{Error, ErrorKind}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::time::{Duration, Instant}; use futures::{StreamExt, TryStreamExt}; use log::{debug, error, log_enabled, trace, Level}; use reqwest::header::CONTENT_ENCODING; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use std::collections::{HashMap, HashSet}; +use std::io::{Error, ErrorKind}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::fs::File; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio_util::io::StreamReader; use url::Url; -use shared::error::create_tuliprox_error_result; -use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind}; -use shared::model::{InputFetchMethod, DEFAULT_USER_AGENT}; use crate::model::{format_elapsed_time, AppConfig, InputSource, ReverseProxyDisabledHeaderConfig}; -use crate::model::{ConfigInput}; -use crate::repository::storage::{get_input_storage_path}; +use crate::model::ConfigInput; +use crate::repository::storage::get_input_storage_path; use crate::repository::storage_const; use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; use crate::utils::{async_file_reader, async_file_writer, debug_if_enabled, IO_BUFFER_SIZE}; -use shared::utils::{filter_request_header, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP}; use crate::utils::{get_file_path, persist_file}; +use shared::error::create_tuliprox_error_result; +use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind}; +use shared::model::{InputFetchMethod, DEFAULT_USER_AGENT}; +use shared::utils::{filter_request_header, human_readable_byte_size, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP}; +use crate::api::model::persist_pipe_stream::tee_dyn_reader; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum MimeCategory { @@ -138,13 +140,63 @@ pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSourc } } + +pub async fn get_input_text_content_as_stream(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option) -> Result { + debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url)); + + if input.url.parse::().is_ok() { + match download_text_content_as_stream(client, None, input, None, persist_filepath).await { + Ok((content, _response_url)) => Ok(content), + Err(e) => { + error!("Failed to download input '{}': {}", &input.name, sanitize_sensitive_info(e.to_string().as_str())); + create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to download") + } + } + } else { + let result = match get_file_path(working_dir, Some(PathBuf::from(&input.url))) { + Some(filepath) => { + if filepath.exists() { + match get_local_file_content_as_stream(&filepath).await { + Ok(content) => { + if persist_filepath.is_some() { + let tee_reader: DynReader = if let Some(path) = persist_filepath { + let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| { + debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); + }))).await; + Box::pin(tee) + } else { + content + }; + Some(tee_reader) + } else { + Some(content) + } + }, + Err(err) => { + return create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed : {}", err); + } + } + } else { + None + } + } + None => None + }; + result.map_or_else(|| { + let msg = format!("cant read input url: {}", sanitize_sensitive_info(&input.url)); + error!("{msg}"); + create_tuliprox_error_result!(TuliproxErrorKind::Notify, "{msg}") + }, Ok) + } +} + pub fn get_client_request - (client: &reqwest::Client, - method: InputFetchMethod, - headers: Option<&HashMap>, - url: &Url, - custom_headers: Option<&HashMap, S>>, - disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> reqwest::RequestBuilder { +(client: &reqwest::Client, + method: InputFetchMethod, + headers: Option<&HashMap>, + url: &Url, + custom_headers: Option<&HashMap, S>>, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> reqwest::RequestBuilder { let request = match method { InputFetchMethod::GET => client.get(url.clone()), InputFetchMethod::POST => { @@ -233,6 +285,26 @@ pub async fn get_local_file_content(file_path: &Path) -> Result Result { + // open file + let file = File::open(file_path).await.map_err(|err| { + std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display())) + })?; + + let mut buf_reader = async_file_reader(file); + + // Peek first 2 Bytes, for gzip detection + let buffer = buf_reader.fill_buf().await?; + let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]); + + if is_gzipped { + // use Async Gzip Decoder + Ok(Box::pin(async_compression::tokio::bufread::GzipDecoder::new(buf_reader))) + } else { + Ok(Box::pin(buf_reader)) + } +} + // pub fn get_local_file_content_blocking(file_path: &PathBuf) -> Result { // match fs::read(file_path) { // Ok(content) => decode_local_file_bytes(content).await, @@ -283,16 +355,23 @@ async fn get_remote_content_as_file(client: &reqwest::Client, input: &ConfigInpu } } -type DynReader = Pin>; +pub type DynReader = Pin>; #[allow(clippy::implicit_hasher)] pub async fn get_remote_content_as_stream( client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, url: &Url, - method: InputFetchMethod, - headers: Option<&HashMap> + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, ) -> Result<(DynReader, String), Error> { - let request = get_client_request(client, method, headers, url, None, None); + let custom_headers = headers.map(|h| { + h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>() + }); + let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers); + let headers: HashMap = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect(); + + let request = get_client_request(client, input.method, Some(&headers), url, None, None); let response = request.send().await.map_err(std::io::Error::other)?; if !response.status().is_success() { @@ -331,12 +410,7 @@ pub async fn get_remote_content_as_stream( async fn get_remote_content(client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> { let start_time = Instant::now(); - let custom_headers = headers.map(|h| { - h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()}); - let merged = get_request_headers(Some(&input.headers), custom_headers.as_ref(), disabled_headers); - let headers: HashMap = merged.iter().map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect(); - - let (mut stream, response_url) = get_remote_content_as_stream(client, url, input.method, Some(&headers)).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?; + let (mut stream, response_url) = get_remote_content_as_stream(client, input, headers, url, disabled_headers).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?; let mut content = String::new(); stream.read_to_string(&mut content).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?; debug_if_enabled!("Request took: {} {}", format_elapsed_time(start_time.elapsed().as_secs()), sanitize_sensitive_info(url.as_str())); @@ -397,10 +471,50 @@ pub async fn download_text_content( Err(err) => Err(err), } } else { - Err(str_to_io_error(&format!( - "Malformed URL {}", - sanitize_sensitive_info(&input.url) - ))) + Err(str_to_io_error(&format!("Malformed URL {}",sanitize_sensitive_info(&input.url)))) + } +} + +pub async fn download_text_content_as_stream( + client: &reqwest::Client, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, +) -> Result<(DynReader, String), Error> { + if let Ok(url) = input.url.parse::() { + let result = if url.scheme() == "file" { + match url.to_file_path() { + Ok(file_path) => get_local_file_content_as_stream(&file_path).await.map(|c| (c, url.to_string())), + Err(()) => Err(str_to_io_error(&format!( + "Unknown file {}", + sanitize_sensitive_info(&input.url) + ))), + } + } else { + get_remote_content_as_stream(client, input, headers, &url, disabled_headers).await + }; + match result { + Ok((content, response_url)) => { + if persist_filepath.is_some() { + + let tee_reader: DynReader = if let Some(path) = persist_filepath { + let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| { + debug!("Persisted {size} bytes"); + }))).await; + Box::pin(tee) + } else { + content + }; + Ok((tee_reader, response_url)) + } else { + Ok((content, response_url)) + } + } + Err(err) => Err(err), + } + } else { + Err(str_to_io_error(&format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) } } @@ -424,6 +538,20 @@ pub async fn get_input_json_content(client: &reqwest::Client, disabled_headers: } } +async fn download_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { + debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url)); + match download_text_content_as_stream(client, disabled_headers, input, None, persist_filepath).await { + Ok((reader, _response_url)) => Ok(reader), + Err(err) => Err(err) + } +} + +pub async fn get_input_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { + match download_json_content_as_stream(client, disabled_headers, input, persist_filepath).await { + Ok(stream) => Ok(stream), + Err(e) => create_tuliprox_error_result!(TuliproxErrorKind::Notify, "cant download input {}, url: {} => {}", input.name, sanitize_sensitive_info(&input.url), sanitize_sensitive_info(e.to_string().as_str())) + } +} pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder { let config = cfg.config.load(); @@ -433,7 +561,6 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder { .pool_max_idle_per_host(10) .danger_accept_invalid_certs(config.accept_insecure_ssl_certificates); - if let Some(proxy_cfg) = config.proxy.as_ref() { match Url::parse(&proxy_cfg.url) { Ok(mut url) => { @@ -451,7 +578,7 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder { Ok(p) => { client = client.proxy(p); } Err(err) => error!("Failed to create SOCKS proxy {url}: {err}"), } - }, + } "http" | "https" => { match reqwest::Proxy::all(url.as_str()) { Ok(p) => { diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index 005345bd0..3c713b27b 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -235,14 +235,14 @@ pub async fn get_xtream_playlist(cfg: &Arc, client: &reqwest::Client, in let stream_file_path = crate::utils::prepare_file_path(input.persist.as_deref(), working_dir, format!("{stream}_").as_str()); match futures::join!( - request::get_input_json_content(client, None, &input_source_category, category_file_path), - request::get_input_json_content(client, None, &input_source_stream, stream_file_path) + request::get_input_json_content_as_stream(client, None, &input_source_category, category_file_path), + request::get_input_json_content_as_stream(client, None, &input_source_stream, stream_file_path) ) { (Ok(category_content), Ok(stream_content)) => { match xtream::parse_xtream(input, *xtream_cluster, - &category_content, - &stream_content) { + category_content, + stream_content).await { Ok(sub_playlist_parsed) => { if let Some(mut xtream_sub_playlist) = sub_playlist_parsed { playlist_groups.append(&mut xtream_sub_playlist); diff --git a/shared/src/utils/serde_utils.rs b/shared/src/utils/serde_utils.rs index 080f55f56..d795df04a 100644 --- a/shared/src/utils/serde_utils.rs +++ b/shared/src/utils/serde_utils.rs @@ -1,7 +1,7 @@ use crate::error::to_io_error; use chrono::{NaiveDateTime, ParseError, TimeZone, Utc}; use serde::de::DeserializeOwned; -use serde::Deserialize; +use serde::{Deserialize, Deserializer}; use serde_json::Value; use std::io; @@ -55,43 +55,52 @@ where }) } -pub fn deserialize_number_from_string<'de, D, T: DeserializeOwned + std::str::FromStr>( - deserializer: D, -) -> Result, D::Error> + +pub fn deserialize_number_from_string<'de, D, T>(deserializer: D) -> Result, D::Error> where - D: serde::Deserializer<'de>, + D: Deserializer<'de>, + T: DeserializeOwned + std::str::FromStr, { - // we define a local enum type inside of the function - // because it is untagged, serde will deserialize as the first variant - // that it can - #[derive(Deserialize)] - #[serde(untagged)] - enum MaybeNumber { - // if it can be parsed as Option, it will be - Value(Option), - // otherwise try parsing as a string - NumberString(String), - } + let raw: Value = Value::deserialize(deserializer)?; - // deserialize into local enum - let value: MaybeNumber = Deserialize::deserialize(deserializer)?; - match value { - // if parsed as T or None, return that - MaybeNumber::Value(value) => Ok(value), + match raw { + // Null → None + Value::Null => Ok(None), - // (if it is any other string) - MaybeNumber::NumberString(s) => { + // its a number + Value::Number(n) => { + let s = n.to_string(); + match s.parse::() { + Ok(v) => Ok(Some(v)), + Err(_) => Ok(None), // Fehler ignorieren, None zurückgeben + } + } + + // String → extract first number + Value::String(s) => { let s = s.trim(); if s.is_empty() { return Ok(None); } - // parse string to number, if fails return None - if let Ok(num) = s.parse::() { - return Ok(Some(num)); + + // find the number + let digits = s.chars() + .skip_while(|c| !c.is_ascii_digit()) + .take_while(|c| c.is_ascii_digit()) + .collect::(); + + if digits.is_empty() { + return Ok(None); } - serde_json::from_str::(s).map_or_else(|_| Ok(None), |val| Ok(Some(val))) + match digits.parse::() { + Ok(v) => Ok(Some(v)), + Err(_) => Ok(None), + } } + + // invalid -> return None + _ => Ok(None), } } From 2738f8b7f3621f5e74b06bd4a68fa7f7858c61ba Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 12 Dec 2025 15:49:10 +0100 Subject: [PATCH 09/11] xtream and m3u parsing with streaming --- CHANGELOG.md | 1 + backend/src/model/xmltv.rs | 1 - backend/src/utils/network/request.rs | 15 +++++---------- shared/src/utils/serde_utils.rs | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0c9d804a..742064f78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ - UserTable: Copy credentials to clipboard from user table - UserTable: Kick user action from streams table - UserTable: Auto-generated username/password for new proxy users +- Update process uses now streams for data processing. # 3.1.7 (2025-10-10) - Added Dark/Bright theme switch diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index 138bceb6b..1b6883b51 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -138,7 +138,6 @@ impl Epg { let inner = writer.get_mut(); inner.flush().await?; - inner.shutdown().await?; Ok(()) } diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 6e3e14627..ccc5ae22d 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -158,16 +158,11 @@ pub async fn get_input_text_content_as_stream(client: &reqwest::Client, input: & if filepath.exists() { match get_local_file_content_as_stream(&filepath).await { Ok(content) => { - if persist_filepath.is_some() { - let tee_reader: DynReader = if let Some(path) = persist_filepath { - let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| { - debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); - }))).await; - Box::pin(tee) - } else { - content - }; - Some(tee_reader) + if let Some(path) = persist_filepath { + let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| { + debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); + }))).await; + Some(Box::pin(tee)) } else { Some(content) } diff --git a/shared/src/utils/serde_utils.rs b/shared/src/utils/serde_utils.rs index d795df04a..4e581de36 100644 --- a/shared/src/utils/serde_utils.rs +++ b/shared/src/utils/serde_utils.rs @@ -72,7 +72,7 @@ where let s = n.to_string(); match s.parse::() { Ok(v) => Ok(Some(v)), - Err(_) => Ok(None), // Fehler ignorieren, None zurückgeben + Err(_) => Ok(None), // Ignore parse errors, return None } } @@ -83,17 +83,21 @@ where return Ok(None); } - // find the number - let digits = s.chars() - .skip_while(|c| !c.is_ascii_digit()) - .take_while(|c| c.is_ascii_digit()) - .collect::(); - - if digits.is_empty() { + // find the number, preserving optional sign + let trimmed = s.trim_start_matches(|c: char| !c.is_ascii_digit() && c != '-' && c != '+'); + if trimmed.is_empty() { return Ok(None); } - match digits.parse::() { + let number_str = trimmed.chars() + .take_while(|c| c.is_ascii_digit() || (*c == '-' || *c == '+') && trimmed.starts_with(|x: char| x == '-' || x == '+')) + .collect::(); + + if number_str.is_empty() || number_str == "-" || number_str == "+" { + return Ok(None); + } + + match number_str.parse::() { Ok(v) => Ok(Some(v)), Err(_) => Ok(None), } From 73017363cc943175ae6a808bd6117b43fdecb8c2 Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 12 Dec 2025 16:10:08 +0100 Subject: [PATCH 10/11] xtream and m3u parsing with streaming --- backend/src/utils/network/request.rs | 26 ++++------ shared/src/utils/serde_utils.rs | 72 +++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index ccc5ae22d..7cc61c9b7 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -13,8 +13,9 @@ use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio_util::io::StreamReader; use url::Url; -use crate::model::{format_elapsed_time, AppConfig, InputSource, ReverseProxyDisabledHeaderConfig}; +use crate::api::model::persist_pipe_stream::tee_dyn_reader; use crate::model::ConfigInput; +use crate::model::{format_elapsed_time, AppConfig, InputSource, ReverseProxyDisabledHeaderConfig}; use crate::repository::storage::get_input_storage_path; use crate::repository::storage_const; use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; @@ -24,7 +25,6 @@ use shared::error::create_tuliprox_error_result; use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind}; use shared::model::{InputFetchMethod, DEFAULT_USER_AGENT}; use shared::utils::{filter_request_header, human_readable_byte_size, sanitize_sensitive_info, short_hash, ENCODING_DEFLATE, ENCODING_GZIP}; -use crate::api::model::persist_pipe_stream::tee_dyn_reader; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum MimeCategory { @@ -162,11 +162,11 @@ pub async fn get_input_text_content_as_stream(client: &reqwest::Client, input: & let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| { debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); }))).await; - Some(Box::pin(tee)) + Some(tee) } else { Some(content) } - }, + } Err(err) => { return create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed : {}", err); } @@ -227,7 +227,7 @@ pub fn get_request_headers(request_header for (key, value) in custom { let key_lc = key.to_lowercase(); if filter_request_header(key_lc.as_str()) { - if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key)) { + if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key_lc.as_str())) { continue; } if header_keys.contains(key_lc.as_str()) { @@ -466,7 +466,7 @@ pub async fn download_text_content( Err(err) => Err(err), } } else { - Err(str_to_io_error(&format!("Malformed URL {}",sanitize_sensitive_info(&input.url)))) + Err(str_to_io_error(&format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) } } @@ -491,16 +491,10 @@ pub async fn download_text_content_as_stream( }; match result { Ok((content, response_url)) => { - if persist_filepath.is_some() { - - let tee_reader: DynReader = if let Some(path) = persist_filepath { - let tee = tee_dyn_reader(content, &path, Some(Arc::new(|size| { - debug!("Persisted {size} bytes"); - }))).await; - Box::pin(tee) - } else { - content - }; + if let Some(path) = persist_filepath { + let tee_reader: DynReader = tee_dyn_reader(content, &path, Some(Arc::new(|size| { + debug!("Persisted {size} bytes"); + }))).await; Ok((tee_reader, response_url)) } else { Ok((content, response_url)) diff --git a/shared/src/utils/serde_utils.rs b/shared/src/utils/serde_utils.rs index 4e581de36..6fddf11c9 100644 --- a/shared/src/utils/serde_utils.rs +++ b/shared/src/utils/serde_utils.rs @@ -83,24 +83,70 @@ where return Ok(None); } - // find the number, preserving optional sign - let trimmed = s.trim_start_matches(|c: char| !c.is_ascii_digit() && c != '-' && c != '+'); - if trimmed.is_empty() { + // Find first digit; include optional sign immediately before it (ignoring whitespace). + let mut last_non_ws: Option<(usize, char)> = None; + let mut digit_pos: Option = None; + for (i, c) in s.char_indices() { + if c.is_ascii_digit() { + digit_pos = Some(i); + break; + } + if !c.is_whitespace() { + last_non_ws = Some((i, c)); + } + } + let Some(digit_i) = digit_pos else { return Ok(None); }; + + let start = match last_non_ws { + Some((i, '-')) | Some((i, '+')) => i, + _ => digit_i, + }; + let mut it = s[start..].chars().peekable(); + + // optional sign + let mut out = String::new(); + if matches!(it.peek(), Some('-' | '+')) { + out.push(it.next().unwrap()); + while matches!(it.peek(), Some(c) if c.is_whitespace()) { + it.next(); + } + } + + // digits + optional single dot + let mut saw_digit = false; + let mut saw_dot = false; + while let Some(&c) = it.peek() { + if c.is_ascii_digit() { + saw_digit = true; + out.push(c); + it.next(); + continue; + } + if c == '.' && !saw_dot { + saw_dot = true; + out.push(c); + it.next(); + continue; + } + break; + } + if !saw_digit { return Ok(None); } - let number_str = trimmed.chars() - .take_while(|c| c.is_ascii_digit() || (*c == '-' || *c == '+') && trimmed.starts_with(|x: char| x == '-' || x == '+')) - .collect::(); - - if number_str.is_empty() || number_str == "-" || number_str == "+" { - return Ok(None); + // Try full parse; if it fails and we included '.', fall back to integer part. + if let Ok(v) = out.parse::() { + return Ok(Some(v)); } - - match number_str.parse::() { - Ok(v) => Ok(Some(v)), - Err(_) => Ok(None), + if saw_dot { + let int_part = out.split('.').next().unwrap_or(""); + if !int_part.is_empty() && int_part != "-" && int_part != "+" { + if let Ok(v) = int_part.parse::() { + return Ok(Some(v)); + } + } } + Ok(None) } // invalid -> return None From 12f8326cb3d424d50fd9d11a9f6b49858b6cd5c4 Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 12 Dec 2025 16:49:13 +0100 Subject: [PATCH 11/11] xtream and m3u parsing with streaming --- backend/src/model/xtream.rs | 2 +- backend/src/processing/parser/xtream.rs | 3 +++ shared/src/utils/serde_utils.rs | 25 +++++++++++++++---------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/backend/src/model/xtream.rs b/backend/src/model/xtream.rs index f740b2eb9..ef747262e 100644 --- a/backend/src/model/xtream.rs +++ b/backend/src/model/xtream.rs @@ -31,7 +31,7 @@ impl XtreamCategory { } } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct XtreamStream { #[serde(default, deserialize_with = "deserialize_as_string")] pub name: String, diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 40e8074f9..ab07cc403 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -237,6 +237,9 @@ mod tests { let reader = Box::pin(async_file_reader(tokio::fs::File::open("/tmp/vod_streams.json").await?)); match map_to_xtream_streams(XtreamCluster::Video, reader).await { Ok(_streams) => { + println!("{:?}", _streams.get(1)); + println!("{:?}", _streams.get(100)); + println!("{:?}", _streams.get(200)); assert!(true); }, Err(err) => { diff --git a/shared/src/utils/serde_utils.rs b/shared/src/utils/serde_utils.rs index 6fddf11c9..840a0d2e4 100644 --- a/shared/src/utils/serde_utils.rs +++ b/shared/src/utils/serde_utils.rs @@ -1,6 +1,5 @@ use crate::error::to_io_error; use chrono::{NaiveDateTime, ParseError, TimeZone, Utc}; -use serde::de::DeserializeOwned; use serde::{Deserialize, Deserializer}; use serde_json::Value; use std::io; @@ -59,7 +58,7 @@ where pub fn deserialize_number_from_string<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, - T: DeserializeOwned + std::str::FromStr, + T: std::str::FromStr, { let raw: Value = Value::deserialize(deserializer)?; @@ -70,9 +69,10 @@ where // its a number Value::Number(n) => { let s = n.to_string(); - match s.parse::() { - Ok(v) => Ok(Some(v)), - Err(_) => Ok(None), // Ignore parse errors, return None + if let Ok(r) = s.parse::() { + Ok(Some(r)) + } else { + Ok(None) } } @@ -83,23 +83,28 @@ where return Ok(None); } - // Find first digit; include optional sign immediately before it (ignoring whitespace). + // Find first digit OR a '.' that is immediately followed by a digit; + // include optional sign immediately before it (ignoring whitespace). let mut last_non_ws: Option<(usize, char)> = None; - let mut digit_pos: Option = None; + let mut num_pos: Option = None; for (i, c) in s.char_indices() { if c.is_ascii_digit() { - digit_pos = Some(i); + num_pos = Some(i); + break; + } + if c == '.' && s[i + 1..].chars().next().is_some_and(|n| n.is_ascii_digit()) { + num_pos = Some(i); break; } if !c.is_whitespace() { last_non_ws = Some((i, c)); } } - let Some(digit_i) = digit_pos else { return Ok(None); }; + let Some(num_i) = num_pos else { return Ok(None); }; let start = match last_non_ws { Some((i, '-')) | Some((i, '+')) => i, - _ => digit_i, + _ => num_i, }; let mut it = s[start..].chars().peekable();