diff --git a/CHANGELOG.md b/CHANGELOG.md index a5e8adc6b..4ab837a11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,32 @@ proxy: username: uname # <- optional basic auth password: secret # <- optional basic auth ``` +- Added support for regular expression-based sequence sorting. + You can now sort both groups and channels using custom regex sequences. +```yaml +sort: + groups: + order: asc + sequence: + - '^Freetv' + - '^Shopping' + - '^Entertainment' + - '^Sunrise' + channels: + - field: caption + group_pattern: '^Freetv' + order: asc + sequence: + - '(?P.*?)\bUHD\b' + - '(?P.*?)\bFHD\b' + - '(?P.*?)\bHD\b' + - '(?P.*?)\bSD\b' +``` +In the example above, groups are sorted based on the specified sequence. +Channels within the `Freetv` group are first sorted by `quality` (as matched by the regex sequence), and then by the `captured prefix`. + +To sort by specific parts of the content, use named capture groups such as `c1`, `c2`, `c3`, etc. +The numeric suffix indicates the priority: `c1` is evaluated first, followed by `c2`, and so on. # 2.2.5 (2025-03-27) - fixed web ui playlist regexp search diff --git a/src/model/config.rs b/src/model/config.rs index 03e4de82e..929c3a729 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -3,13 +3,13 @@ use bitflags::bitflags; use enum_iterator::Sequence; use std::borrow::BorrowMut; use std::collections::{HashMap, HashSet}; +use std::fmt; use std::fmt::Display; use std::fs::File; use std::io::BufRead; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use std::fmt; use tokio::sync::RwLock; use crate::auth::user::UserCredential; @@ -240,6 +240,18 @@ impl ProcessTargets { } } +fn compile_regex_vec(patterns: Option<&Vec>) -> Result>, M3uFilterError> { + patterns.as_ref() + .map(|seq| { + seq.iter() + .map(|s| Regex::new(s).map_err(|err| { + create_m3u_filter_error!(M3uFilterErrorKind::Info, "cant parse regex: {} {err}", s) + })) + .collect::, _>>() + }) + .transpose() // convert Option> to Result> +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct ConfigSortGroup { @@ -247,7 +259,16 @@ pub struct ConfigSortGroup { #[serde(default, skip_serializing_if = "Option::is_none")] pub sequence: Option>, #[serde(default, skip)] - pub t_sequence: Option>, + pub t_sequence: Option>, +} + + +impl ConfigSortGroup { + pub fn prepare(&mut self) -> Result<(), M3uFilterError> { + // Compile sequence patterns, if any + self.t_sequence = compile_regex_vec(self.sequence.as_ref())?; + Ok(()) + } } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -276,16 +297,7 @@ impl ConfigSortChannel { ); // Compile sequence patterns, if any - self.t_sequence = self.sequence.as_ref() - .map(|seq| { - seq.iter() - .map(|s| Regex::new(s).map_err(|err| { - create_m3u_filter_error!(M3uFilterErrorKind::Info, "cant parse regex: {} {err}", s) - })) - .collect::, _>>() - }) - .transpose()?; // convert Option> to Result> - + self.t_sequence = compile_regex_vec(self.sequence.as_ref())?; Ok(()) } } @@ -303,6 +315,9 @@ pub struct ConfigSort { impl ConfigSort { pub fn prepare(&mut self) -> Result<(), M3uFilterError> { + if let Some(group) = self.groups.as_mut() { + group.prepare()?; + } if let Some(channels) = self.channels.as_mut() { handle_m3u_filter_error_result_list!(M3uFilterErrorKind::Info, channels.iter_mut().map(ConfigSortChannel::prepare)); } @@ -1780,18 +1795,18 @@ impl ConfigProxy { let uname = username.trim(); let pwd = password.trim(); if uname.is_empty() || pwd.is_empty() { - return Err(M3uFilterError::new(M3uFilterErrorKind::Info,"Proxy credentials missing".to_string())); + return Err(M3uFilterError::new(M3uFilterErrorKind::Info, "Proxy credentials missing".to_string())); } self.username = Some(uname.to_string()); self.password = Some(pwd.to_string()); } else { - return Err(M3uFilterError::new(M3uFilterErrorKind::Info,"Proxy credentials missing".to_string())); + return Err(M3uFilterError::new(M3uFilterErrorKind::Info, "Proxy credentials missing".to_string())); } } self.url = self.url.trim().to_string(); if self.url.is_empty() { - return Err(M3uFilterError::new(M3uFilterErrorKind::Info,"Proxy url missing".to_string())); + return Err(M3uFilterError::new(M3uFilterErrorKind::Info, "Proxy url missing".to_string())); } Ok(()) } diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index 78f658526..841ad0c1f 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -1,35 +1,35 @@ -use crate::{Config}; -use crate::model::config::{ConfigInput, ConfigRename}; +use crate::model::config::{ConfigInput, ConfigRename, SortOrder}; use crate::utils::network::epg; use crate::utils::network::m3u; use crate::utils::network::xtream; +use crate::Config; use core::cmp::Ordering; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -use std::sync::{Arc}; -use tokio::sync::Mutex; +use std::sync::Arc; use std::thread; +use tokio::sync::Mutex; -use log::{debug, error, info, log_enabled, trace, warn, Level}; -use std::time::Instant; -use deunicode::deunicode; use crate::foundation::filter::{get_field_value, set_field_value, MockValueProcessor, ValueProvider}; -use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, get_errors_notify_message, notify_err}; +use crate::m3u_filter_error::{get_errors_notify_message, notify_err, M3uFilterError, M3uFilterErrorKind}; use crate::messaging::{send_message, MsgKind}; use crate::model::config::{ConfigSortChannel, ConfigSortGroup, ConfigTarget, InputType, - ItemField, ProcessTargets, ProcessingOrder, SortOrder::{Asc, Desc}}; + ItemField, ProcessTargets, ProcessingOrder}; use crate::model::mapping::{CounterModifier, Mapping, MappingValueProcessor}; use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor, PlaylistEntry, PlaylistGroup, PlaylistItem, UUIDType, XtreamCluster}; use crate::model::stats::{InputStats, PlaylistStats, SourceStats, TargetStats}; -use crate::processing::processor::affix::apply_affixes; use crate::processing::playlist_watch::process_group_watch; +use crate::processing::processor::affix::apply_affixes; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::persist_playlist; use crate::utils::default_utils::default_as_default; -use crate::utils::{debug_if_enabled}; +use crate::utils::debug_if_enabled; +use deunicode::deunicode; +use log::{debug, error, info, log_enabled, trace, warn, Level}; +use std::time::Instant; -use crate::model::xmltv::{Epg}; +use crate::model::xmltv::Epg; use crate::processing::parser::xmltv::flatten_tvguide; use crate::processing::processor::epg::process_playlist_epg; use crate::utils::step_measure::StepMeasure; @@ -59,55 +59,115 @@ fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt Some(new_playlist) } -fn playlistgroup_comparator(a: &PlaylistGroup, b: &PlaylistGroup, group_sort: &ConfigSortGroup, match_as_ascii: bool) -> Ordering { - let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.to_string() }; - let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.to_string() }; - let ordering = value_a.partial_cmp(&value_b).unwrap(); - match group_sort.order { - Asc => ordering, - Desc => ordering.reverse() +fn playlist_comparator( + sequence: Option<&Vec>, + order: &SortOrder, + value_a: &str, + value_b: &str, +) -> Ordering { + if let Some(regex_list) = sequence { + let mut match_a = None; + let mut match_b = None; + + for (i, regex) in regex_list.iter().enumerate() { + if match_a.is_none() { + if let Some(caps) = regex.captures(value_a) { + match_a = Some((i, caps)); + } + } + if match_b.is_none() { + if let Some(caps) = regex.captures(value_b) { + match_b = Some((i, caps)); + } + } + + // If both matches found → break + if match_a.is_some() && match_b.is_some() { + break; + } + } + + match (match_a, match_b) { + (Some((idx_a, caps_a)), Some((idx_b, caps_b))) => { + // Different regex indices → sort by their sequence order. + if idx_a != idx_b { + return match order { + SortOrder::Asc => idx_a.cmp(&idx_b), + SortOrder::Desc => idx_b.cmp(&idx_a), + }; + } + + // Same regex → sort by captures (c1, c2, …) + let mut named: Vec<_> = regex_list[idx_a] + .capture_names() + .flatten() + .filter(|name| name.starts_with('c')) + .collect(); + + named.sort_by_key(|name| name[1..].parse::().unwrap_or(0)); + + for name in named { + let va = caps_a.name(name).map(|m| m.as_str()); + let vb = caps_b.name(name).map(|m| m.as_str()); + if let (Some(va), Some(vb)) = (va, vb) { + let o = va.cmp(vb); + if o != Ordering::Equal { + return match order { + SortOrder::Asc => o, + SortOrder::Desc => o.reverse(), + }; + } + } + } + + Ordering::Equal + } + (Some(_), None) => match order { + SortOrder::Asc => Ordering::Less, + SortOrder::Desc => Ordering::Greater, + }, + (None, Some(_)) => match order { + SortOrder::Asc => Ordering::Greater, + SortOrder::Desc => Ordering::Less, + }, + (None, None) => { + // NP match → fallback + let o = value_a.cmp(value_b); + match order { + SortOrder::Asc => o, + SortOrder::Desc => o.reverse(), + } + } + } + } else { + // No Regex-Sequence defined → fallback + let o = value_a.cmp(value_b); + match order { + SortOrder::Asc => o, + SortOrder::Desc => o.reverse(), + } } } -fn playlistitem_comparator(a: &PlaylistItem, b: &PlaylistItem, channel_sort: &ConfigSortChannel, match_as_ascii: bool) -> Ordering { +fn playlistgroup_comparator(a: &PlaylistGroup, b: &PlaylistGroup, group_sort: &ConfigSortGroup, match_as_ascii: bool) -> Ordering { + let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.to_string() }; + let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.to_string() }; + + playlist_comparator(group_sort.t_sequence.as_ref(), &group_sort.order, &value_a, &value_b) +} + +fn playlistitem_comparator( + a: &PlaylistItem, + b: &PlaylistItem, + channel_sort: &ConfigSortChannel, + match_as_ascii: bool, +) -> Ordering { let raw_value_a = get_field_value(a, &channel_sort.field); let raw_value_b = get_field_value(b, &channel_sort.field); let value_a = if match_as_ascii { deunicode(&raw_value_a) } else { raw_value_a }; let value_b = if match_as_ascii { deunicode(&raw_value_b) } else { raw_value_b }; - channel_sort.sequence.as_ref().map_or_else(|| { - let ordering = value_a.partial_cmp(&value_b).unwrap(); - match channel_sort.order { - Asc => ordering, - Desc => ordering.reverse() - } - }, |custom_order| { - // Check indices in the custom order vector - let index_a = custom_order.iter().position(|s| s == &value_a); - let index_b = custom_order.iter().position(|s| s == &value_b); - match (index_a, index_b) { - (Some(idx_a), Some(idx_b)) => { - // Both items found in custom order, compare indices - idx_a.cmp(&idx_b) - } - (Some(_), None) => { - // Only 'a' found in custom order, it comes first - Ordering::Less - } - (None, Some(_)) => { - // Only 'b' found in custom order, it comes first - Ordering::Greater - } - (None, None) => { - // Neither found, fall back to default ordering - let ordering = value_a.partial_cmp(&value_b).unwrap(); - match channel_sort.order { - Asc => ordering, - Desc => ordering.reverse(), - } - } - } - }) + playlist_comparator(channel_sort.t_sequence.as_ref(), &channel_sort.order, &value_a, &value_b) } fn sort_playlist(target: &ConfigTarget, new_playlist: &mut [PlaylistGroup]) { @@ -133,7 +193,7 @@ fn sort_playlist(target: &ConfigTarget, new_playlist: &mut [PlaylistGroup]) { fn channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { let assigned_chnos: HashSet = new_playlist.iter().flat_map(|g| &g.channels) .filter(|c| !c.header.chno.is_empty()) - .map(|c|c.header.chno.as_str()) + .map(|c| c.header.chno.as_str()) .flat_map(str::parse::).collect(); let mut chno = 1; for group in new_playlist { @@ -209,11 +269,10 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem { let ref_chan = &mut channel; let mut mock_processor = MockValueProcessor {}; for m in &mapping.mapper { - let provider = ValueProvider { pli: &ref_chan.clone() }; + let provider = ValueProvider { pli: &ref_chan.clone() }; let mut processor = MappingValueProcessor { pli: ref_chan, mapper: m }; match &m.t_filter { Some(filter) => { - if filter.filter(&provider, &mut mock_processor) { apply_pattern!(&m.t_pattern, &provider, &mut processor); } @@ -360,7 +419,7 @@ async fn process_source(client: Arc, cfg: Arc, source_i } let elapsed = start_time.elapsed().as_secs(); input_stats.insert(input_name.to_string(), create_input_stat(group_count, channel_count, error_list.len(), - input.input_type, input_name, elapsed)); + input.input_type, input_name, elapsed)); } } if source_playlists.is_empty() { @@ -428,7 +487,7 @@ async fn process_sources(client: Arc, config: Arc, user let handles = &mut handle_list; let process = move || { // TODO better way ? - let rt = tokio::runtime::Runtime::new().unwrap(); + let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { let (input_stats, target_stats, mut res_errors) = process_source(Arc::clone(&http_client), cfg, index, usr_trgts).await; shared_errors.lock().await.append(&mut res_errors); @@ -572,7 +631,6 @@ async fn process_playlist_for_target(client: Arc, } fn process_epg(processed_fetched_playlists: &mut Vec) -> (Vec, Vec) { - let mut new_playlist = vec![]; let mut new_epg = vec![]; @@ -625,16 +683,47 @@ pub async fn exec_processing(client: Arc, cfg: Arc, tar #[cfg(test)] mod tests { - #[test] - fn test() { - let data = [("yessport5", "heyessport5gold"), ("yessport5", "heyesport5gold")]; + // #[test] + // fn test_jaro_winkeler() { + // let data = [("yessport5", "heyessport5gold"), ("yessport5", "heyesport5gold")]; + // + // data.iter().for_each(|(first, second)| + // println!("jaro_winkler {} = {} => {}", first, second, strsim::jaro_winkler(first, second))); + // // println!("jaro {}", strsim::jaro(data.0, data.1)); + // // println!("levenhstein {}", strsim::levenshtein(data.0, data.1)); + // // println!("damerau_levenshtein {:?}", strsim::damerau_levenshtein(data.0, data.1)); + // // println!("osa distance {:?}", strsim::osa_distance(data.0, data.1)); + // // println!("sorensen dice {:?}", strsim::sorensen_dice(data.0, data.1)); + // } - data.iter().for_each(|(first, second)| - println!("jaro_winkler {} = {} => {}", first, second, strsim::jaro_winkler(first, second))); - // println!("jaro {}", strsim::jaro(data.0, data.1)); - // println!("levenhstein {}", strsim::levenshtein(data.0, data.1)); - // println!("damerau_levenshtein {:?}", strsim::damerau_levenshtein(data.0, data.1)); - // println!("osa distance {:?}", strsim::osa_distance(data.0, data.1)); - // println!("sorensen dice {:?}", strsim::sorensen_dice(data.0, data.1)); + use crate::model::config::{ConfigSortChannel, ItemField, SortOrder}; + use crate::model::playlist::{PlaylistItem, PlaylistItemHeader}; + use crate::processing::processor::playlist::playlistitem_comparator; + use regex::Regex; + + #[test] + fn test_sort() { + let mut channels: Vec = vec![ + ("D", "HD"), ("A", "FHD"), ("Z", "HD"), ("K", "HD"), ("B", "HD"), ("A", "HD"), + ("K", "UHD"), ("C", "HD"), ("L", "FHD"), ("R", "UHD"), ("T", "SD"), ("A", "FHD"), + ].into_iter().map(|(name, quality)| PlaylistItem { header: PlaylistItemHeader { title: format!("Chanel {name} [{quality}]"), ..Default::default() } }).collect::>().into(); + + let channel_sort = ConfigSortChannel { + field: ItemField::Caption, + group_pattern: ".*".to_string(), + order: SortOrder::Asc, + sequence: None, + t_sequence: Some(vec![ + Regex::new(r"(?P.*?)\bUHD\b").unwrap(), + Regex::new(r"(?P.*?)\bFHD\b").unwrap(), + Regex::new(r"(?P.*?)\bHD\b").unwrap(), + ]), + t_re_group_pattern: Some(Regex::new(".*").unwrap()), + }; + + channels.sort_by(|chan1, chan2| playlistitem_comparator(chan1, chan2, &channel_sort, true)); + let expected = vec!["Chanel K [UHD]", "Chanel R [UHD]", "Chanel A [FHD]", "Chanel A [FHD]", "Chanel L [FHD]", "Chanel A [HD]", "Chanel B [HD]", "Chanel C [HD]", "Chanel D [HD]", "Chanel K [HD]", "Chanel Z [HD]", "Chanel T [SD]"]; + let sorted = channels.into_iter().map(|pli| pli.header.title.clone()).collect::>(); + assert_eq!(expected, sorted); } } \ No newline at end of file