From 6f776782f124490d56e3bb6572d014a56abe90f3 Mon Sep 17 00:00:00 2001 From: euzu Date: Thu, 8 Jan 2026 15:47:34 +0100 Subject: [PATCH] Playlist Update Optimization: Reduced Memory Usage An optimization has been introduced to reduce memory consumption during playlist updates. Overview Previously, provider playlists were fully loaded into memory during the update process. For large playlists (e.g. hundreds of thousands of entries or multiple providers), this could result in significant RAM usage. With the new implementation, it is now possible to optionally read provider playlists directly from disk instead of keeping them entirely in memory. How It Works - users can configure whether: - Provider playlists are loaded into memory (previous behavior), or - Provider playlists are streamed to/from disk to minimize RAM usage. Processing remains sequential and batch-based, ensuring identical functional behavior. This approach significantly lowers peak memory usage, especially on systems with limited resources. Benefits - Reduced peak RAM consumption during playlist updates - Better scalability for large playlists and multiple providers - Full backward compatibility Trade-offs / Drawbacks - Increased processing time due to reduced in-memory caching - Higher disk I/O usage, especially for large or fragmented playlists - Performance depends more strongly on disk speed (Nvme SSD, HDD) - Slightly increased CPU overhead due to repeated parsing and deserialization - Not optimal for environments where fast updates are more important than memory usage Recommendation - Use in-memory mode for systems with sufficient RAM and a focus on update speed - Use disk-based mode for large playlists, multiple providers, or memory-constrained environments --- backend/src/api/api_utils.rs | 9 +++--- backend/src/api/endpoints/hdhomerun_api.rs | 22 +++++++------ backend/src/api/endpoints/hls_api.rs | 2 +- backend/src/api/endpoints/m3u_api.rs | 2 +- backend/src/api/endpoints/xmltv_api.rs | 2 +- backend/src/api/endpoints/xtream_api.rs | 32 +++++++++---------- backend/src/api/model/model_utils.rs | 5 ++- .../api/model/streams/persist_pipe_stream.rs | 2 +- backend/src/model/config/api_proxy.rs | 2 +- backend/src/processing/parser/m3u.rs | 3 +- backend/src/processing/processor/playlist.rs | 22 +++++++------ backend/src/processing/processor/sort.rs | 18 +++++++++-- backend/src/repository/bplustree.rs | 11 +++---- .../src/repository/m3u_playlist_iterator.rs | 4 +-- backend/src/repository/playlist_source.rs | 2 +- backend/src/repository/storage.rs | 4 +-- backend/src/repository/strm_repository.rs | 8 ++--- backend/src/repository/user_repository.rs | 2 +- backend/src/repository/xtream_repository.rs | 4 +-- backend/src/tools/lru_cache.rs | 2 +- backend/src/utils/file/config_reader.rs | 6 ++-- backend/src/utils/network/epg.rs | 4 +-- backend/src/utils/network/request.rs | 16 +++++----- backend/src/utils/network/xtream.rs | 4 +-- .../components/config/library_config_view.rs | 3 ++ frontend/src/services/requests.rs | 2 +- shared/src/foundation/filter.rs | 6 ++-- shared/src/model/config/api_proxy.rs | 4 +-- shared/src/model/config/epg_smart_match.rs | 2 +- shared/src/model/config/rename.rs | 2 +- shared/src/model/config/sort.rs | 4 +-- shared/src/model/config/video_download.rs | 2 +- shared/src/model/stream_info.rs | 2 ++ shared/src/utils/string_utils.rs | 24 ++++++++------ 34 files changed, 132 insertions(+), 107 deletions(-) diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index f4817bc94..34b0f0d8a 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -41,6 +41,7 @@ use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tokio::sync::Mutex; use tokio_util::io::ReaderStream; use url::Url; +use shared::concat_string; const CONTENT_TYPE_BIN: &str = "application/cbor"; @@ -548,7 +549,7 @@ async fn create_stream_response_details( let provider_handle = if stream.is_none() { let provider_handle = streaming_strategy.provider_handle.take(); app_state.connection_manager.release_provider_handle(provider_handle).await; - error!("Cant open stream {}", sanitize_sensitive_info(&request_url)); + error!("Can't open stream {}", sanitize_sensitive_info(&request_url)); None } else { streaming_strategy.provider_handle.take() @@ -597,9 +598,9 @@ where // if there is an action_path (like for timeshift duration/start), it will be added in front of the stream_id if self.action_path.is_empty() { - format!("{provider_id}{extension}") + concat_string!(&provider_id.to_string(), &extension) } else { - format!("{}/{provider_id}{extension}", trim_slash(self.action_path)) + concat_string!(&trim_slash(self.action_path), "/", &provider_id.to_string(), &extension) } } } @@ -671,7 +672,7 @@ where &provider_url, ) { None => { - error!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", params.req_context); + error!("Can't find stream url for target {target_name}, context {}, stream_id {virtual_id}", params.req_context); return Some(axum::http::StatusCode::BAD_REQUEST.into_response()); } Some(url) => { diff --git a/backend/src/api/endpoints/hdhomerun_api.rs b/backend/src/api/endpoints/hdhomerun_api.rs index 441e4b6df..0b3513dbd 100644 --- a/backend/src/api/endpoints/hdhomerun_api.rs +++ b/backend/src/api/endpoints/hdhomerun_api.rs @@ -133,11 +133,12 @@ where url: stream_url, }; match serde_json::to_string(&lineup) { - Ok(content) => Ok(Bytes::from(if has_next { - format!("{content},") - } else { - content - })), + Ok(mut content) => { + if has_next { + content.push(','); + } + Ok(Bytes::from(content)) + }, Err(_) => Ok(Bytes::from("")), } }); @@ -165,11 +166,12 @@ where .clone(), }; match serde_json::to_string(&lineup) { - Ok(content) => Ok(Bytes::from(if has_next { - format!("{content},") - } else { - content - })), + Ok(mut content) => { + if has_next { + content.push(','); + } + Ok(Bytes::from(content)) + }, Err(_) => Ok(Bytes::from("")), } }); diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 7e5de296d..192074150 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -236,7 +236,7 @@ async fn hls_api_stream( let input = try_option_bad_request!( app_state.app_config.get_input_by_id(params.input_id), true, - format!("Cant find input {} for target {target_name}, stream_id {virtual_id}, hls", params.input_id) + format!("Can't find input {} for target {target_name}, stream_id {virtual_id}, hls", params.input_id) ); debug_if_enabled!("ID chain for hls endpoint: request_stream_id={} -> virtual_id={virtual_id}", params.stream_id); diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index d8d8545bd..48c643a26 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -128,7 +128,7 @@ async fn m3u_api_stream( .app_config .get_input_by_name(&pli.input_name), true, - format!("Cant find input {} for target {target_name}, stream_id {virtual_id}", pli.input_name) + format!("Can't find input {} for target {target_name}, stream_id {virtual_id}", pli.input_name) ); if pli.item_type.is_local() { diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index b05be20d5..b88dccc73 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -81,7 +81,7 @@ fn get_epg_path_for_target_of_type(target_name: &str, epg_path: PathBuf) -> Opti return Some(epg_path); } trace!( - "Cant find epg file for {target_name} target: {}", + "Can't find epg file for {target_name} target: {}", epg_path.to_str().unwrap_or("?") ); None diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index a4be32d06..f36e8510c 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -252,7 +252,7 @@ async fn xtream_player_api_stream( let input = try_option_bad_request!( app_state.app_config.get_input_by_name(&pli.input_name), true, - format!( "Cant find input {} for target {target_name}, context {}, stream_id {virtual_id}", pli.input_name, stream_req.context) + format!( "Can't find input {} for target {target_name}, context {}, stream_id {virtual_id}", pli.input_name, stream_req.context) ); if pli.item_type.is_local() { @@ -368,7 +368,7 @@ async fn xtream_player_api_stream( get_xtream_player_api_stream_url(&input, stream_req.context, &query_path, session_url), true, format!( - "Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", + "Can't find stream url for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context ) ); @@ -445,7 +445,7 @@ async fn xtream_player_api_stream_with_token( .get_input_by_name(&pli.input_name), true, format!( - "Cant find input {} for target {target_name}, context {}, stream_id {}", + "Can't find input {} for target {target_name}, context {}, stream_id {}", pli.input_name, stream_req.context, pli.virtual_id ) ); @@ -529,7 +529,7 @@ async fn xtream_player_api_stream_with_token( ), true, format!( - "Cant find stream url for target {target_name}, context {}, stream_id {}", + "Can't find stream url for target {target_name}, context {}, stream_id {}", stream_req.context, virtual_id ) ); @@ -973,7 +973,7 @@ async fn xtream_get_short_epg( } } } - warn!("Cant find short epg with id: {target_name}/{stream_id}"); + warn!("Can't find short epg with id: {target_name}/{stream_id}"); get_empty_epg_response().into_response() } @@ -1285,7 +1285,7 @@ async fn xtream_player_api( ) .await ), - _ => Some(info_err_res!("Cant find content: {action} for target: {}", &target.name)), + _ => Some(info_err_res!("Can't find content: {action} for target: {}", &target.name)), }; match result { @@ -1319,7 +1319,7 @@ async fn xtream_player_api( } } else { match (user_target.is_none(), api_req.action.is_empty()) { - (true, _) => debug!("Cant find user!"), + (true, _) => debug!("Can't find user!"), (_, true) => debug!("Parameter action is empty!"), _ => debug!("Bad request!"), } @@ -1331,16 +1331,14 @@ fn xtream_create_content_stream( xtream_iter: impl Iterator, ) -> impl Stream> { stream::once(async { Ok::(Bytes::from("[")) }).chain( - stream::iter(xtream_iter.map(move |(line, has_next)| { - Ok::(Bytes::from(if has_next { - format!("{line},") - } else { - line.clone() - })) - })) - .chain(stream::once(async { - Ok::(Bytes::from("]")) - })), + stream::iter(xtream_iter.map(move |(mut line, has_next)| { + if has_next { + line.push(','); + } + Ok::(Bytes::from(line)) + })).chain(stream::once(async { + Ok::(Bytes::from("]")) + })), ) } diff --git a/backend/src/api/model/model_utils.rs b/backend/src/api/model/model_utils.rs index f20c2ea29..f3e406f5b 100644 --- a/backend/src/api/model/model_utils.rs +++ b/backend/src/api/model/model_utils.rs @@ -5,13 +5,12 @@ use reqwest::header::HeaderMap; use shared::utils::{filter_response_header}; pub fn get_response_headers(headers: &HeaderMap) -> Vec<(String, String)> { - let response_headers: Vec<(String, String)> = headers.iter() + headers.iter() .filter(|(key, _)| filter_response_header(key.as_str())) .filter_map(|(key, value)| { value.to_str().ok().map(|v| (key.to_string(), v.to_string())) }) - .collect(); - response_headers + .collect() } pub fn get_stream_response_with_headers(custom: Option<(Vec<(String, String)>, StatusCode)>) -> (axum::http::StatusCode, axum::http::HeaderMap) { diff --git a/backend/src/api/model/streams/persist_pipe_stream.rs b/backend/src/api/model/streams/persist_pipe_stream.rs index 32a8c2081..3ee9a13fe 100644 --- a/backend/src/api/model/streams/persist_pipe_stream.rs +++ b/backend/src/api/model/streams/persist_pipe_stream.rs @@ -88,7 +88,7 @@ pub async fn tee_dyn_reader( let file = match tokio::fs::File::create(persist_path).await { Ok(f) => f, Err(err) => { - error!("Cant open file to write: {}, {err}", persist_path.display()); + error!("Can't open file to write: {}, {err}", persist_path.display()); return reader; } }; diff --git a/backend/src/model/config/api_proxy.rs b/backend/src/model/config/api_proxy.rs index d885266bc..00caca04c 100644 --- a/backend/src/model/config/api_proxy.rs +++ b/backend/src/model/config/api_proxy.rs @@ -129,7 +129,7 @@ impl ApiProxyConfig { } else { let user_db_path = get_api_user_db_path(cfg); if let Ok(true) = tokio::fs::try_exists(&user_db_path).await { - // we cant have user defined in db file. + // we can't have user defined in db file. // we need to load them and save them into the config file if let Ok(stored_users) = load_api_user(cfg).await { for stored_user in stored_users { diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index c12bf95c8..902e17954 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -5,6 +5,7 @@ use shared::utils::{default_supported_video_extensions, extract_id_from_url, Str use std::borrow::BorrowMut; use tokio::io::AsyncBufReadExt; use indexmap::IndexMap; +use shared::concat_string; // other implementations like calculating text_distance on all titles took too much time // we keep it now as simple as possible and less memory intensive. @@ -13,7 +14,7 @@ fn get_title_group(text: &str) -> String { let parts = alphabetic_only.split_whitespace(); let mut combination = String::new(); for p in parts { - combination = format!("{combination} {p}").trim().to_string(); + combination = concat_string!(&combination, " " , p); if combination.len() > 2 { return combination; } diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 0d53e8203..7eba638f9 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -560,17 +560,19 @@ impl PlaylistProcessingContext { pub async fn get_input_lock(&self, input_name: &str) -> OwnedRwLockWriteGuard<()> { let mut locks = self.input_locks.lock().await; - // Clean up stale weak references + // Try to upgrade the existing weak reference + let lock = locks.get(input_name) + .and_then(Weak::upgrade) + .unwrap_or_else(|| { + let new_lock = Arc::new(RwLock::new(())); + locks.insert(input_name.to_string(), Arc::downgrade(&new_lock)); + new_lock + }); + + // Clean up stale references periodically locks.retain(|_, weak| weak.strong_count() > 0); - if let Some(weak) = locks.get(input_name) { - if let Some(strong) = weak.upgrade() { - return strong.write_owned().await; - } - } - - let lock = Arc::new(RwLock::new(())); - locks.insert(input_name.to_string(), Arc::downgrade(&lock)); + drop(locks); // Release mutex before awaiting write lock lock.write_owned().await } } @@ -776,7 +778,7 @@ pub fn process_favourites(playlist: &mut Vec, favourites_cfg: Opt let mut fav_groups: IndexMap, Vec> = IndexMap::new(); for pg in playlist.iter() { for pli in &pg.channels { - // series episodes cant be included in favourites + // series episodes can't be included in favourites if pli.header.item_type == PlaylistItemType::Series || pli.header.item_type == PlaylistItemType::LocalSeries { continue; } diff --git a/backend/src/processing/processor/sort.rs b/backend/src/processing/processor/sort.rs index 4f22c99fd..1d21f51a8 100644 --- a/backend/src/processing/processor/sort.rs +++ b/backend/src/processing/processor/sort.rs @@ -109,10 +109,22 @@ fn playlist_comparator( } fn playlistgroup_comparator(a: &PlaylistGroup, b: &PlaylistGroup, group_sort: &ConfigSortGroup, match_as_ascii: bool) -> Ordering { - let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.to_string() }; - let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.to_string() }; + let ascii_a; + let ascii_b; + let value_a: &str = if match_as_ascii { + ascii_a = deunicode(&a.title); + &ascii_a + } else { + &a.title + }; + let value_b: &str = if match_as_ascii { + ascii_b = deunicode(&b.title); + &ascii_b + } else { + &b.title + }; - playlist_comparator(group_sort.sequence.as_ref(), group_sort.order, &value_a, &value_b) + playlist_comparator(group_sort.sequence.as_ref(), group_sort.order, value_a, value_b) } fn playlistitem_comparator( diff --git a/backend/src/repository/bplustree.rs b/backend/src/repository/bplustree.rs index d3c4a9852..179fec81e 100644 --- a/backend/src/repository/bplustree.rs +++ b/backend/src/repository/bplustree.rs @@ -2477,17 +2477,16 @@ where for (k, v) in query.iter() { let value_bytes = binary_serialize(&v)?; - let len = value_bytes.len(); - let offset = current_offset; // Write value header (flag + payload) - write_buffer.write_all(&[COMPRESSION_FLAG_NONE])?; - write_buffer.write_all(&value_bytes)?; + let (flag, payload) = compress_if_beneficial(&value_bytes); + write_buffer.write_all(&[flag])?; + write_buffer.write_all(&payload)?; - current_offset += 1 + len as u64; + let stored_len = u32::try_from(1 + payload.len()).map_err(to_io_error)?; + current_offset += u64::from(stored_len); - let stored_len = u32::try_from(len + 1).map_err(to_io_error)?; value_infos.push((k, ValueInfo { mode: ValueStorageMode::Single(offset), length: stored_len, diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index d47865856..8fbadf756 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -94,13 +94,13 @@ impl M3uPlaylistIterator { shared::concat_string!( cap = cap; &self.base_url, "/", prefix_path, "/", stream_type, "/", - &self.username, "/", &self.password, "/", m3u_pli.virtual_id + &self.username, "/", &self.password, "/", &m3u_pli.virtual_id.to_string() ) } else { shared::concat_string!( cap = cap; &self.base_url, "/", prefix_path, "/", - &self.username, "/", &self.password, "/", m3u_pli.virtual_id + &self.username, "/", &self.password, "/", &m3u_pli.virtual_id.to_string() ) } } diff --git a/backend/src/repository/playlist_source.rs b/backend/src/repository/playlist_source.rs index c16be6390..fb37e3183 100644 --- a/backend/src/repository/playlist_source.rs +++ b/backend/src/repository/playlist_source.rs @@ -339,7 +339,7 @@ macro_rules! impl_single_file_disk_source { fn update_playlist<'a>(&'a mut self, _plg: &'a PlaylistGroup) -> BoxFuture<'a, ()> { Box::pin(async move { - warn!("update_playlist should not be called for M3U Disk playlist"); + warn!("update_playlist should not be called for Disk playlist"); }) } diff --git a/backend/src/repository/storage.rs b/backend/src/repository/storage.rs index 7e242cb06..4564cf246 100644 --- a/backend/src/repository/storage.rs +++ b/backend/src/repository/storage.rs @@ -4,7 +4,7 @@ use crate::utils; use shared::error::notify_err; use shared::error::TuliproxError; use std::path::{Path, PathBuf}; -use shared::notify_err_res; +use shared::{concat_string, notify_err_res}; pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { // Join directly with &str to avoid an intermediate PathBuf allocation @@ -32,7 +32,7 @@ pub fn get_input_storage_path(input_name: &str, working_dir: &str) -> std::io::R let sanitized_name: String = input_name.chars() .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect(); - let name = format!("input_{sanitized_name}"); + let name = concat_string!(cap = 6 + sanitized_name.len(); "input_", &sanitized_name); let path = Path::new(working_dir).join(name); // Create the directory and return the path or propagate the error std::fs::create_dir_all(&path).map(|()| path) diff --git a/backend/src/repository/strm_repository.rs b/backend/src/repository/strm_repository.rs index 1e9230140..8576b7184 100644 --- a/backend/src/repository/strm_repository.rs +++ b/backend/src/repository/strm_repository.rs @@ -12,7 +12,7 @@ use filetime::{set_file_times, FileTime}; use log::{error, trace}; use regex::Regex; use serde::Serialize; -use shared::error::{TuliproxError, info_err_res, info_err}; +use shared::error::{TuliproxError, info_err_res}; use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType}; use shared::utils::{ arc_str_serde, extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS}; use std::collections::{HashMap, HashSet, VecDeque}; @@ -764,7 +764,7 @@ pub async fn write_strm_playlist( continue; // skip creation } - // if we cant create the directory skip this entry + // if we can't create the directory skip this entry if !ensure_strm_file_directory(&mut failed, &output_path).await { continue; } @@ -797,7 +797,7 @@ pub async fn write_strm_playlist( if failed.is_empty() { Ok(()) } else { - Err(info_err!("{}", failed.join(", "))) + info_err_res!("{}", failed.join(", ")) } } async fn write_strm_index_file( @@ -990,7 +990,7 @@ impl DirNode { } /// Because of rust ownership we don't want to use References or Mutexes. -/// Because of async operations ve cant use recursion. +/// Because of async operations ve can't use recursion. /// We use paths identifier to handle the tree construction. /// Rust sucks!!! async fn build_directory_tree(root_path: &Path) -> HashMap { diff --git a/backend/src/repository/user_repository.rs b/backend/src/repository/user_repository.rs index 81942ba47..a2bc42474 100644 --- a/backend/src/repository/user_repository.rs +++ b/backend/src/repository/user_repository.rs @@ -52,7 +52,7 @@ impl StoredProxyUserCredentialsDeprecated { // This is a Helper class to store all user into one Database file. // For the Config files we keep the old structure where a user is assigned to a target. // But for storing inside one db file it is easier to store the target next to the user. -// due to known issue with bincode and skip_serialization_if we have to list all fields and cant use ProxyUserCredentials +// due to known issue with bincode and skip_serialization_if we have to list all fields and can't use ProxyUserCredentials #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] struct StoredProxyUserCredentials { pub target: String, diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index 6702069e3..6753a029d 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -167,7 +167,7 @@ fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Optio pub type CategoryKey = (XtreamCluster, Arc); -// Because interner is not thread safe we cant use it currently for interning. +// Because interner is not thread safe we can't use it currently for interning. // We leave the argument for later optimizations. async fn load_old_category_ids(path: &Path, _interner: &mut StringInterner) -> (u32, HashMap) { let old_path = path.to_path_buf(); @@ -378,7 +378,7 @@ pub fn xtream_get_collection_path( return Ok(col_path); } } - Err(string_to_io_error(format!("Cant find collection: {target_name}/{collection_name}"))) + Err(string_to_io_error(format!("Can't find collection: {target_name}/{collection_name}"))) } async fn xtream_read_item_for_stream_id( diff --git a/backend/src/tools/lru_cache.rs b/backend/src/tools/lru_cache.rs index 62c30f114..50cd31c26 100644 --- a/backend/src/tools/lru_cache.rs +++ b/backend/src/tools/lru_cache.rs @@ -69,7 +69,7 @@ impl LRUResourceCache { }; let file_size = usize::try_from(metadata.len()).unwrap_or(0); - // we need to duplicate because of closure we cant call insert_to_cache + // we need to duplicate because of closure we can't call insert_to_cache { // insert_to_cache let mut path = self.cache_dir.clone(); diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index 9f73999d1..04047007b 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -59,7 +59,7 @@ pub async fn read_api_proxy_config(config: &AppConfig, resolve_env: bool) -> Res } Ok(Some(api_proxy)) } else { - warn!("cant read api_proxy_config file: {api_proxy_file_path}"); + warn!("can't read api_proxy_config file: {api_proxy_file_path}"); Ok(None) } } @@ -279,13 +279,13 @@ pub fn read_api_proxy_file(api_proxy_file: &str, resolve_env: bool) -> Result { if resolve_env { if let Err(err) = api_proxy_dto.prepare() { - exit!("cant read api-proxy-config file: {err}"); + exit!("can't read api-proxy-config file: {err}"); } } Ok(Some(api_proxy_dto)) } Err(err) => { - info_err_res!("cant read api-proxy-config file: {err}") + info_err_res!("can't read api-proxy-config file: {err}") } } }) diff --git a/backend/src/utils/network/epg.rs b/backend/src/utils/network/epg.rs index 89c54adc6..6bed93d54 100644 --- a/backend/src/utils/network/epg.rs +++ b/backend/src/utils/network/epg.rs @@ -37,7 +37,7 @@ async fn download_epg_file(url: &str, ctx: &PlaylistProcessingContext, input: &C if let Ok(elapsed) = std::time::SystemTime::now().duration_since(modified) { if elapsed.as_secs() < input.cache_duration_seconds { debug!("Using cached epg file: {}", persist_file_path.display()); - return Ok(persist_file_path.clone()); + return Ok(persist_file_path); } } } @@ -49,7 +49,7 @@ async fn download_epg_file(url: &str, ctx: &PlaylistProcessingContext, input: &C let _input_lock = ctx.get_input_lock(&lock_key); if ctx.is_input_downloaded(&lock_key).await { - return Ok(persist_file_path.clone()); + return Ok(persist_file_path); } match request::get_input_epg_content_as_file(&ctx.client, input, working_dir, url, &persist_file_path).await { diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index fdbfd7055..91bf5eb02 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -68,7 +68,7 @@ pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &Con match download_epg_content_as_file(client, input, url_str, persist_filepath).await { Ok(content) => Ok(content), Err(e) => { - error!("cant download input {} epg url: {} => {}", input.name, sanitize_sensitive_info(url_str), sanitize_sensitive_info(e.to_string().as_str())); + error!("can't download input {} epg url: {} => {}", input.name, sanitize_sensitive_info(url_str), sanitize_sensitive_info(e.to_string().as_str())); notify_err_res!("Failed to download") } } @@ -77,7 +77,7 @@ pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &Con Some(filepath) => { if filepath.exists() { if let Err(e) = tokio::fs::copy(&filepath, persist_filepath).await { - error!("cant persist to: {} => {}", persist_filepath.display(), e); + error!("can't persist to: {} => {}", persist_filepath.display(), e); return notify_err_res!("Failed to persist: {} => {}", persist_filepath.display(), e); } if filepath.exists() { @@ -93,7 +93,7 @@ pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &Con }; result.map_or_else(|| { - let msg = format!("cant read input url: {}", sanitize_sensitive_info(url_str)); + let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); error!("{msg}"); notify_err_res!("{msg}") }, Ok) @@ -118,7 +118,7 @@ pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSourc if let Some(persist_file_value) = persist_filepath { let to_file = &persist_file_value; if let Err(e) = tokio::fs::copy(&filepath, to_file).await { - error!("cant persist to: {} => {}", to_file.to_str().unwrap_or("?"), e); + error!("can't persist to: {} => {}", to_file.to_str().unwrap_or("?"), e); return notify_err_res!("Failed to persist: {} => {}", to_file.to_str().unwrap_or("?"), e); } } @@ -136,7 +136,7 @@ pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSourc None => None }; result.map_or_else(|| { - let msg = format!("cant read input url: {}", sanitize_sensitive_info(&input.url)); + let msg = format!("can't read input url: {}", sanitize_sensitive_info(&input.url)); error!("{msg}"); notify_err_res!("{msg}") }, Ok) @@ -181,7 +181,7 @@ pub async fn get_input_text_content_as_stream(client: &reqwest::Client, input: & None => None }; result.map_or_else(|| { - let msg = format!("cant read input url: {}", sanitize_sensitive_info(&input.url)); + let msg = format!("can't read input url: {}", sanitize_sensitive_info(&input.url)); error!("{msg}"); notify_err_res!("{msg}") }, Ok) @@ -539,7 +539,7 @@ async fn download_json_content(client: &reqwest::Client, disabled_headers: Optio pub async fn get_input_json_content(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option, trace_log: bool) -> Result { match download_json_content(client, disabled_headers, input, persist_filepath, trace_log).await { Ok(content) => Ok(content), - Err(e) => notify_err_res!("cant download input {}, => {}", input.name, sanitize_sensitive_info(e.to_string().as_str())) + Err(e) => notify_err_res!("can't download input {}, => {}", input.name, sanitize_sensitive_info(e.to_string().as_str())) } } @@ -554,7 +554,7 @@ async fn download_json_content_as_stream(client: &reqwest::Client, disabled_head pub async fn get_input_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option) -> Result { match download_json_content_as_stream(client, disabled_headers, input, persist_filepath).await { Ok(stream) => Ok(stream), - Err(e) => notify_err_res!("cant download input {} => {}", input.name, sanitize_sensitive_info(e.to_string().as_str())) + Err(e) => notify_err_res!("can't download input {} => {}", input.name, sanitize_sensitive_info(e.to_string().as_str())) } } diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index e2a69772d..fe0dc56d0 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -147,7 +147,7 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, let config = &app_state.app_config.config.load(); match get_target_storage_path(config, target.name.as_str()) { None => { - error!("Failed to get target storage path {}. Cant save episodes", &target.name); + error!("Failed to get target storage path {}. Can't save episodes", &target.name); } Some(target_path) => { let mut in_memory_updates = Vec::new(); @@ -224,7 +224,7 @@ pub async fn get_xtream_stream_info(client: &reqwest::Client, } } - Err(string_to_io_error(format!("Cant find stream with id: {}/{}/{}", + Err(string_to_io_error(format!("Can't find stream with id: {}/{}/{}", target.name.replace(' ', "_").as_str(), &cluster, pli.get_virtual_id()))) } diff --git a/frontend/src/app/components/config/library_config_view.rs b/frontend/src/app/components/config/library_config_view.rs index 8de8b1a54..8ada299d7 100644 --- a/frontend/src/app/components/config/library_config_view.rs +++ b/frontend/src/app/components/config/library_config_view.rs @@ -245,6 +245,9 @@ pub fn LibraryConfigView() -> Html { let form_state = form_state.clone(); Callback::from(move |idx: usize| { let mut current_list = form_state.form.scan_directories.clone(); + if idx >= current_list.len() { + return; + } current_list.remove(idx); form_state.dispatch(LibraryConfigFormAction::ScanDirectories(current_list)); }) diff --git a/frontend/src/services/requests.rs b/frontend/src/services/requests.rs index 2fb978b22..64a60b7e2 100644 --- a/frontend/src/services/requests.rs +++ b/frontend/src/services/requests.rs @@ -35,7 +35,7 @@ pub fn set_token(token: Option<&str>) { const DUMMY_TOKEN: &str = "eyJraWQiOiJkZWZhdWx0IiwiYWxnIjoiUlMyNTYifQ.eyJsb2dpbiI6ImR1bW15In0.WWzZP0hICmJeIgMLVYNOpayriEC08J_lYssk9z8GglHXfZ6oJUDv3svlJDA8sQG025VA_LR5UzyyiWeQDCdpWyrCI_nI2Xd-3ga3JwWtxHE9NWFalgq0Q9jjxoB4LYWCXsAkqoZqk6s7b3F5Fi_h5oYHfwM4h8hXEbrgnJ_Z1wpSc7HNh6SUnOllxcaJOxYlRrlUn3XulSSf2NhHe3XotvFguiIV1-RIns3cSIL29bvMUEFw84w7BfJn-joynZsWlfJBvzyOiuDqduXa0deH7b962unM2wPpbvTgliJhFFOUBHClRhBOmoo0cijuMZB4K7NjgjGmU5eVfHG6pVWs_b0ikS4V_P6RJcNS6Alcc_HB_YXv0yCD3pjcBbuRXAskivEhgXuecdRMGQgohAhXplLuu5SR0K6Bcrt7UFFnBi2qN6fbw1i4s8PDXqiTu4rIg9agCkVNfplRvj8Szl6egF0Vd1TN1WGEarkdINEUyfNQkAihFY5BKxfaPun1-a0VydRMZElu6VzrrUMxXt4T7zybuJZI63C3mKLEHZixdSC76c9AE-zGom5LZYE4mqwd4dW3QHtWFZgGZiL9C_VBIf63WzjTYhWVuO2U8O9bsKkSEl5L-Ww9j8ccDHp5nc7y6yUgSYd600TBRI7WblFovLsl2tjElvUqfJZhj6JmX_Q"; // The Authorization header is for the Backend Authenticator mandatory. -// If we don't set a dummy token the backend api cant be called. +// If we don't set a dummy token the backend api can't be called. pub fn check_dummy_token() { if get_token().is_none() { set_token(Some(DUMMY_TOKEN)); diff --git a/shared/src/foundation/filter.rs b/shared/src/foundation/filter.rs index 308ab77b2..5c6cedf4e 100644 --- a/shared/src/foundation/filter.rs +++ b/shared/src/foundation/filter.rs @@ -163,7 +163,7 @@ fn get_caption<'a>(provider: &'a ValueProvider<'a>, rewc: &'a CompiledRegex) -> } } - if let Some(value) = provider.get("title") { + if let Some(value) = provider.get("name") { if rewc.re.is_match(&value) { return (true, value); } @@ -293,7 +293,7 @@ fn get_parser_regexp( let regstr = apply_templates_to_pattern_single(&parsed_text, templates)?; let re = regex::Regex::new(regstr.as_str()); if re.is_err() { - return info_err_res!("cant parse regex: {}", regstr); + return info_err_res!("can't parse regex: {}", regstr); } let regexp = re.unwrap(); if log_enabled!(Level::Trace) { @@ -344,7 +344,7 @@ fn get_parser_type_comparison(expr: Pair) -> Result let expr_inner = expr.into_inner(); let text_item_type = expr_inner.as_str(); let item_type = get_filter_item_type(text_item_type); - item_type.map_or_else(|| info_err_res!("cant parse item type: {text_item_type}"), + item_type.map_or_else(|| info_err_res!("can't parse item type: {text_item_type}"), |itype| Ok(Filter::TypeComparison(ItemField::Type, itype))) } diff --git a/shared/src/model/config/api_proxy.rs b/shared/src/model/config/api_proxy.rs index 642998f43..af1baced8 100644 --- a/shared/src/model/config/api_proxy.rs +++ b/shared/src/model/config/api_proxy.rs @@ -43,11 +43,11 @@ impl ApiProxyServerInfoDto { } self.protocol = self.protocol.trim().to_string(); if self.protocol.is_empty() { - return info_err_res!("protocol cant be empty for api server config"); + return info_err_res!("protocol can't be empty for api server config"); } self.host = self.host.trim().to_string(); if self.host.is_empty() { - return info_err_res!("host cant be empty for api server config"); + return info_err_res!("host can't be empty for api server config"); } if let Some(port) = self.port.as_ref() { let port = port.trim().to_string(); diff --git a/shared/src/model/config/epg_smart_match.rs b/shared/src/model/config/epg_smart_match.rs index 3aa76d4ec..d97033773 100644 --- a/shared/src/model/config/epg_smart_match.rs +++ b/shared/src/model/config/epg_smart_match.rs @@ -98,7 +98,7 @@ impl EpgSmartMatchConfigDto { if let Some(regstr) = self.normalize_regex.as_ref() { let re = regex::Regex::new(regstr.as_str()); if re.is_err() { - return info_err_res!("cant parse regex: {}", regstr); + return info_err_res!("can't parse regex: {}", regstr); } }; diff --git a/shared/src/model/config/rename.rs b/shared/src/model/config/rename.rs index 2d35f05d0..65822a876 100644 --- a/shared/src/model/config/rename.rs +++ b/shared/src/model/config/rename.rs @@ -14,7 +14,7 @@ impl ConfigRenameDto { pub fn prepare(&mut self, templates: Option<&Vec>) -> Result<(), TuliproxError> { self.pattern = apply_templates_to_pattern_single(&self.pattern, templates)?; if let Err(err) = regex::Regex::new(&self.pattern) { - return info_err_res!("cant parse regex: {} {err}", &self.pattern); + return info_err_res!("can't parse regex: {} {err}", &self.pattern); } Ok(()) } diff --git a/shared/src/model/config/sort.rs b/shared/src/model/config/sort.rs index cf0fb3751..131e87539 100644 --- a/shared/src/model/config/sort.rs +++ b/shared/src/model/config/sort.rs @@ -10,7 +10,7 @@ fn compile_regex_vec(patterns: Option<&Vec>) -> Result .map(|seq| { seq.iter() .map(|s| Regex::new(s).map_err(|err| { - info_err!("cant parse regex: {s} {err}") + info_err!("can't parse regex: {s} {err}") })) .collect::, _>>() }) @@ -106,7 +106,7 @@ impl ConfigSortChannelDto { self.group_pattern = apply_templates_to_pattern_single(&self.group_pattern, templates)?; // Compile group_pattern Regex::new(&self.group_pattern).map_err(|err| { - info_err!("cant parse regex: {} {err}", &self.group_pattern) + info_err!("can't parse regex: {} {err}", &self.group_pattern) })?; // Transform sequence with templates if provided, otherwise use raw sequence diff --git a/shared/src/model/config/video_download.rs b/shared/src/model/config/video_download.rs index 890d0885a..3b3af23c1 100644 --- a/shared/src/model/config/video_download.rs +++ b/shared/src/model/config/video_download.rs @@ -68,7 +68,7 @@ impl VideoConfigDto { if let Some(episode_pattern) = &downl.episode_pattern { if let Err(err) = regex::Regex::new(episode_pattern) { - return info_err_res!("cant parse regex: {episode_pattern} {err}"); + return info_err_res!("can't parse regex: {episode_pattern} {err}"); } } } diff --git a/shared/src/model/stream_info.rs b/shared/src/model/stream_info.rs index 6090cb6de..35d8a9383 100644 --- a/shared/src/model/stream_info.rs +++ b/shared/src/model/stream_info.rs @@ -1,3 +1,4 @@ +use crate::utils::arc_str_serde; use crate::utils::is_blank_optional_string; use std::net::SocketAddr; use std::sync::Arc; @@ -12,6 +13,7 @@ pub struct StreamChannel { pub provider_id: u32, pub item_type: PlaylistItemType, pub cluster: XtreamCluster, + #[serde(with = "arc_str_serde")] pub group: Arc, pub title: String, pub url: String, diff --git a/shared/src/utils/string_utils.rs b/shared/src/utils/string_utils.rs index f5d8950be..4b338f24b 100644 --- a/shared/src/utils/string_utils.rs +++ b/shared/src/utils/string_utils.rs @@ -142,17 +142,23 @@ pub fn longest<'a>(a: &'a str, b: &'a str) -> &'a str { #[macro_export] macro_rules! concat_string { (cap = $cap:expr; $($arg:expr),* $(,)?) => {{ - let mut __s = ::std::string::String::with_capacity($cap); - $( let _ = ::std::fmt::Write::write_fmt(&mut __s, format_args!("{}", $arg)); )* - __s + let mut s = String::with_capacity($cap); + $( s.push_str($arg); )* + s }}; - ( $($arg:expr),* $(,)?) => {{ - let mut __s = ::std::string::String::new(); - $( let _ = ::std::fmt::Write::write_fmt(&mut __s, format_args!("{}", $arg)); )* - __s + ($($s:expr),+ $(,)?) => {{ + let parts = [$($s),+]; + let cap = parts.iter().map(|s| s.len()).sum(); + + let mut out = String::with_capacity(cap); + for s in parts { + out.push_str(s); + } + out }}; } + #[cfg(test)] mod test { use std::collections::HashSet; @@ -179,14 +185,14 @@ mod test { let a = "hello"; let b = String::from("world"); let n = 42; - let s = shared::concat_string!(a, " ", b, " ", n); + let s = shared::concat_string!(a, " ", &b, " ", &n.to_string()); assert_eq!(s, "hello world 42"); } #[test] fn test_concat_string_with_cap() { let part = "abc"; - let s = shared::concat_string!(cap = 16; part, "/", 123); + let s = shared::concat_string!(cap = 16; part, "/", &123.to_string()); assert_eq!(s, "abc/123"); }