From aa73d7b7ad5ec2858b9e362fbcd350d96620e4fc Mon Sep 17 00:00:00 2001 From: euzu Date: Tue, 13 Jan 2026 17:03:03 +0100 Subject: [PATCH] - Refactored Sort - Added cluster to favourites - Added Regex cache to avoid multiple compilations of same regex --- backend/src/model/config/epg_smart_match.rs | 7 +++-- backend/src/processing/processor/playlist.rs | 2 +- .../src/repository/m3u_playlist_iterator.rs | 5 +++- backend/src/repository/m3u_repository.rs | 20 +++++++------ backend/src/repository/sorted_index.rs | 15 +++++++--- backend/src/repository/xtream_repository.rs | 7 +++-- frontend/src/app/components/search.rs | 1 + shared/src/model/config/epg_smart_match.rs | 28 +++++++++++-------- 8 files changed, 54 insertions(+), 31 deletions(-) diff --git a/backend/src/model/config/epg_smart_match.rs b/backend/src/model/config/epg_smart_match.rs index e01b24823..d15d22b99 100644 --- a/backend/src/model/config/epg_smart_match.rs +++ b/backend/src/model/config/epg_smart_match.rs @@ -22,8 +22,11 @@ impl From<&EpgSmartMatchConfigDto> for EpgSmartMatchConfig { Self { enabled: dto.enabled, normalize_regex: match &dto.normalize_regex { - Some(regex_str) => shared::model::REGEX_CACHE.get_or_compile(regex_str).unwrap_or_else(|_| CONSTANTS.re_epg_normalize.clone()), - None => CONSTANTS.re_epg_normalize.clone(), + Some(regex_str) => shared::model::REGEX_CACHE.get_or_compile(regex_str).unwrap_or_else(|e| { + log::warn!("Invalid normalize_regex '{regex_str}': {e}, using default"); + CONSTANTS.re_epg_normalize.clone() + }), + None => CONSTANTS.re_epg_normalize.clone(), }, strip: match &dto.strip { Some(list) => list.iter().map(|s| s.to_lowercase()).collect(), diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 987fb7980..8a1070def 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -206,7 +206,7 @@ fn map_playlist(source: &mut dyn PlaylistSource, target: &ConfigTarget, _interne Box::new(iter.flat_map(move |chan| map_channel_and_flatten(chan, mapping))) as Box> }); - let mut next_groups: IndexMap<(XtreamCluster, Arc), PlaylistGroup> = IndexMap::new(); + let mut next_groups: IndexMap = IndexMap::new(); let mut grp_id: u32 = 0; for channel in mapped_iter { let group_title = channel.header.group.clone(); diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index f572971aa..56aa2ce5b 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -166,7 +166,10 @@ impl M3uPlaylistIterator { } } else { match self.reader.next() { - Some(Ok((_, v))) => Some((v, self.reader.peek().is_some())), + Some(Ok((_, v))) => { + let has_next = matches!(self.reader.peek(), Some(Ok(_))); + Some((v, has_next)) + }, Some(Err(e)) => { error!("Iterator error: {e}"); None diff --git a/backend/src/repository/m3u_repository.rs b/backend/src/repository/m3u_repository.rs index 6a75b0f20..5e7df23b3 100644 --- a/backend/src/repository/m3u_repository.rs +++ b/backend/src/repository/m3u_repository.rs @@ -5,21 +5,21 @@ use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; use crate::repository::m3u_playlist_iterator::M3uPlaylistM3uTextIterator; use crate::repository::storage::{get_file_path_for_db_index, get_target_storage_path}; use crate::repository::storage_const; +use crate::repository::xtream_repository::CategoryKey; use crate::utils; use crate::utils::{async_file_writer, IO_BUFFER_SIZE}; use indexmap::IndexMap; use log::error; -use shared::error::{notify_err, info_err, string_to_io_error, str_to_io_error, TuliproxError}; +use shared::concat_string; +use shared::error::{info_err, notify_err, str_to_io_error, string_to_io_error, TuliproxError}; use shared::model::{M3uPlaylistItem, PlaylistGroup}; use shared::model::{PlaylistItem, PlaylistItemType, XtreamCluster}; -use crate::repository::xtream_repository::CategoryKey; use std::io::Error; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio::task; -use shared::concat_string; macro_rules! cant_write_result { ($path:expr, $err:expr) => { @@ -32,7 +32,7 @@ pub fn m3u_get_file_path_for_db(target_path: &Path) -> PathBuf { } pub fn m3u_get_epg_file_path(target_path: &Path) -> PathBuf { - let path = target_path.join(PathBuf::from(format!("{}.{}", storage_const::FILE_M3U, storage_const::FILE_SUFFIX_DB))); + let path = target_path.join(PathBuf::from(concat_string!(storage_const::FILE_M3U, ".", storage_const::FILE_SUFFIX_DB))); utils::add_prefix_to_filename(&path, "epg_", Some("xml")) } @@ -177,7 +177,10 @@ pub async fn iter_raw_m3u_playlist(config: &AppConfig, target: &ConfigTarget) -> // Re-open query for fallback match BPlusTreeQuery::::try_new(&m3u_path) { Ok(mut query) => query.iter().map(|(_, v)| v).collect(), - Err(_) => Vec::new(), + Err(fallback_err) => { + error!("Fallback query also failed {}: {fallback_err}", m3u_path.display()); + Vec::new() + } } } } @@ -186,14 +189,13 @@ pub async fn iter_raw_m3u_playlist(config: &AppConfig, target: &ConfigTarget) -> }; let len = items.len(); - Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i < len - 1)))) + Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i + 1 < len)))) } Err(_) => None } } pub async fn persist_input_m3u_playlist(app_config: &Arc, m3u_path: &Path, playlist: &[PlaylistGroup]) -> Result<(), TuliproxError> { - let file_lock = app_config.file_locks.write_lock(m3u_path).await; let m3u_path_clone = m3u_path.to_path_buf(); @@ -212,8 +214,8 @@ pub async fn persist_input_m3u_playlist(app_config: &Arc, m3u_path: & tree.store(&m3u_path_clone).map_err(|err| cant_write_result!(&m3u_path_clone, err))?; Ok(()) }) - .await - .map_err(|err| notify_err!("failed to write m3u playlist: {} - {err}", m3u_path.display()))??; + .await + .map_err(|err| notify_err!("failed to write m3u playlist: {} - {err}", m3u_path.display()))??; Ok(()) } diff --git a/backend/src/repository/sorted_index.rs b/backend/src/repository/sorted_index.rs index caf796c19..3e0f59285 100644 --- a/backend/src/repository/sorted_index.rs +++ b/backend/src/repository/sorted_index.rs @@ -192,8 +192,7 @@ where self.writer.flush()?; // Seek back to count position and write final count - let file = self.writer.into_inner()?; - let mut file = file; + let mut file = self.writer.into_inner()?; file.seek(SeekFrom::Start(8))?; // After magic + version file.write_all(&self.count.to_le_bytes())?; file.sync_all()?; @@ -433,9 +432,9 @@ where // Update cache if self.block_cache.len() >= CACHE_CAPACITY { - self.block_cache.shift_remove_index(0); // Remove LRU + self.block_cache.shift_remove_index(0); // FIFO eviction } - self.block_cache.insert(block_offset, buf.clone()); + self.block_cache.insert(block_offset, buf); } let block_buffer = self.block_cache.get(&block_offset).unwrap(); @@ -444,6 +443,14 @@ where let count = u32::from_le_bytes(block_buffer[0..4].try_into().map_err(|e| { io::Error::new(io::ErrorKind::InvalidData, format!("Invalid count: {e}")) })?); + + if u32::from(value_index) >= count { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Value index {value_index} out of bounds (count: {count})"), + )); + } + let mut pos = 4; // Skip to target value diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index faff4c68a..a5fe6e674 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -598,7 +598,10 @@ pub async fn iter_raw_xtream_playlist(app_config: &AppConfig, target: &ConfigTar // Re-open query for fallback match BPlusTreeQuery::::try_new(&xtream_path) { Ok(mut query) => query.iter().map(|(_, v)| v).collect(), - Err(_) => Vec::new(), + Err(fallback_err) => { + error!("Fallback query also failed {}: {fallback_err}", xtream_path.display()); + Vec::new() + } } } } @@ -607,7 +610,7 @@ pub async fn iter_raw_xtream_playlist(app_config: &AppConfig, target: &ConfigTar }; let len = items.len(); - Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i < len - 1)))) + Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i + 1 < len)))) } Err(_) => None } diff --git a/frontend/src/app/components/search.rs b/frontend/src/app/components/search.rs index 1f6dc276a..a438c599a 100644 --- a/frontend/src/app/components/search.rs +++ b/frontend/src/app/components/search.rs @@ -50,6 +50,7 @@ pub fn Search(props: &SearchProps) -> Html { } else { regex_active.set(RegexState::Invalid); } + shared::model::REGEX_CACHE.sweep(); } } } diff --git a/shared/src/model/config/epg_smart_match.rs b/shared/src/model/config/epg_smart_match.rs index f9e65ae70..bf5fda656 100644 --- a/shared/src/model/config/epg_smart_match.rs +++ b/shared/src/model/config/epg_smart_match.rs @@ -1,9 +1,10 @@ +use crate::error::{TuliproxError}; +use crate::info_err; use crate::utils::is_blank_optional_string; -use std::fmt::Display; +use crate::utils::{default_best_match_threshold, default_match_threshold, is_default_best_match_threshold, + is_default_match_threshold, is_false}; use log::warn; -use crate::error::{info_err_res, TuliproxError}; -use crate::utils::{is_false, default_match_threshold, default_best_match_threshold, - is_default_match_threshold, is_default_best_match_threshold}; +use std::fmt::Display; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "lowercase")] @@ -45,9 +46,15 @@ pub struct EpgSmartMatchConfigDto { pub name_prefix_separator: Option>, #[serde(default, skip_serializing_if = "is_false")] pub fuzzy_matching: bool, - #[serde(default = "default_match_threshold", skip_serializing_if = "is_default_match_threshold")] + #[serde( + default = "default_match_threshold", + skip_serializing_if = "is_default_match_threshold" + )] pub match_threshold: u16, - #[serde(default = "default_best_match_threshold", skip_serializing_if = "is_default_best_match_threshold")] + #[serde( + default = "default_best_match_threshold", + skip_serializing_if = "is_default_best_match_threshold" + )] pub best_match_threshold: u16, } impl Default for EpgSmartMatchConfigDto { @@ -66,7 +73,6 @@ impl Default for EpgSmartMatchConfigDto { } impl EpgSmartMatchConfigDto { - /// # Panics /// /// Prepares the EPG smart match configuration by validating thresholds, compiling normalization regex, and setting default values as needed. @@ -96,11 +102,9 @@ impl EpgSmartMatchConfigDto { } if let Some(regstr) = self.normalize_regex.as_ref() { - let re = crate::model::REGEX_CACHE.get_or_compile(regstr.as_str()); - if re.is_err() { - return info_err_res!("can't parse regex: {}", regstr); - } - }; + crate::model::REGEX_CACHE.get_or_compile(regstr.as_str()) + .map_err(|_| info_err!("can't parse regex: {}", regstr))?; + } Ok(()) }