diff --git a/Cargo.lock b/Cargo.lock index 504552e5c..fc7cc578a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3849,6 +3849,7 @@ dependencies = [ "bytes", "chrono", "ciborium", + "deunicode", "enum-iterator", "fastrand", "hex", diff --git a/README.md b/README.md index 329885e79..c96670e82 100644 --- a/README.md +++ b/README.md @@ -1443,23 +1443,19 @@ Has the following top level entries: Is referenced in the `config.yml`, should be a unique identifier ### 2.3.2 `match_as_ascii` -If you have non ascii characters in you playlist and want to -write regexp without considering chars like `é` and use `e` instead, set this option to `true`. -[unidecode](https://crates.io/crates/unidecode) is used to convert the text. +If you have non-ASCII characters in your playlist (e.g., `é`, `ö`, `ß`) and want to write filters without considering these accents (e.g., using `e` to match `é`), set this option to `true`. +The system will automatically deunicode the field values on-the-fly during filtering and mapping operations. -### 2.3.3 `create_alias` -Set this to `true` to keep the original channel and add a mapped copy whenever a mapper rule matches. The copy gets a derived UUID and follows the mapped fields (e.g. a new group), so a channel can live in multiple groups such as favourites. Example: ```yaml mapping: - id: favourites_news match_as_ascii: true - create_alias: true mapper: - filter: 'Group ~ "(?i)news"' script: | - @Group = "Favourites" + add_favourite("Favourites") ``` ### 2.3.4 `mapper` @@ -1498,6 +1494,7 @@ It is whitespace-tolerant and uses familiar programming concepts with a custom s - replace(text, match, replacement) - pad(text | number, number, char, optional position: "<" | ">" | "^") - format(fmt_text, ...args) + - add_favourite(group_name) Field names are: `name`, `title"`, `caption"`, `group"`, `id"`, `chno"`, `logo"`, `logo_small"`, `parent_code"`, `audio_track"`, `time_shift" | "url"`, `epg_channel_id"`, `epg_id`. Format is very simple and only supports in text replacement like `format("Hello {}! Hello {}!", "Bob", "World")` When you use Regular expressions it could be that your match contains multiple results. diff --git a/backend/src/model/mapping.rs b/backend/src/model/mapping.rs index b8d0d6241..dd23f3af1 100644 --- a/backend/src/model/mapping.rs +++ b/backend/src/model/mapping.rs @@ -30,8 +30,6 @@ pub struct Mapping { pub id: String, #[serde(default)] pub match_as_ascii: bool, - #[serde(default)] - pub create_alias: bool, pub mapper: Option>, pub counter: Option>, #[serde(skip_serializing, skip_deserializing)] @@ -45,7 +43,6 @@ impl From<&MappingDto> for Mapping { Self { id: dto.id.clone(), match_as_ascii: dto.match_as_ascii, - create_alias: dto.create_alias, mapper: dto.mapper.as_ref().map(|l| l.iter().map(Mapper::from).collect()), counter: dto.counter.clone(), t_counter: dto.t_counter.clone(), diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 4eecd1745..2596b65e5 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -26,17 +26,16 @@ use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::{persist_input_playlist, persist_playlist}; use crate::utils::StepMeasure; use crate::utils::{debug_if_enabled, trace_if_enabled}; -use deunicode::deunicode; use futures::StreamExt; use log::{debug, error, info, log_enabled, trace, warn, Level}; use shared::error::{get_errors_notify_message, notify_err, TuliproxError}; use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider}; -use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster}; -use shared::utils::{default_as_default, hash_bytes}; +use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster}; +use shared::utils::{create_alias_uuid, default_as_default}; use std::time::Instant; -fn is_valid(pli: &PlaylistItem, filter: &Filter) -> bool { - let provider = ValueProvider { pli }; +fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { + let provider = ValueProvider { pli, match_as_ascii }; filter.filter(&provider) } @@ -46,7 +45,7 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) let mut new_playlist = Vec::with_capacity(128); for pg in playlist.iter_mut() { let channels = pg.channels.iter() - .filter(|&pli| is_valid(pli, filter)).cloned().collect::>(); + .filter(|&pli| is_valid(pli, filter, false)).cloned().collect::>(); trace!("Filtered group {} has now {}/{} items", pg.title, channels.len(), pg.channels.len()); if !channels.is_empty() { new_playlist.push(PlaylistGroup { @@ -60,52 +59,8 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) Some(new_playlist) } -pub fn apply_favourites_to_playlist( - _playlist: &mut [PlaylistGroup], - _favourites_cfg: Option<&[ConfigFavourites]>, -) { - // TODO implement favourites - // if let Some(favourites) = favourites_cfg { - // let mut fav_groups: HashMap> = HashMap::new(); - // - // for pg in playlist.iter_mut() { - // for pli in &pg.channels { - // for fav in favourites { - // if is_valid(pli, &fav.filter) { - // let mut channel = pli.clone(); - // channel.header.copy = true; - // channel.header.group.clone_from(&fav.group); - // channel.header.gen_uuid(); - // fav_groups - // .entry(fav.group.clone()) - // .or_default() - // .push(channel); - // } - // } - // } - // } - // - // for (group_name, channels) in fav_groups { - // if !channels.is_empty() { - // let xtream_cluster = channels[0].header.xtream_cluster; - // playlist.push(PlaylistGroup { - // id: 0, - // title: group_name, - // channels, - // xtream_cluster, - // }); - // } - // } - // } -} - fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { - if let Some(mut filtered_playlist) = apply_filter_to_playlist(playlist, &target.filter) { - apply_favourites_to_playlist(&mut filtered_playlist, target.favourites.as_deref()); - Some(filtered_playlist) - } else { - None - } + apply_filter_to_playlist(playlist, &target.filter) } fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) { @@ -170,53 +125,39 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt } } -fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType { - let mut data = Vec::with_capacity(base_uuid.len() + mapping_id.len()); - data.extend_from_slice(base_uuid.as_ref()); - data.extend_from_slice(mapping_id.as_bytes()); - hash_bytes(&data) -} -fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, bool) { +fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, Vec, bool) { let mut matched = false; + let mut virtual_items = vec![]; if let Some(mapper) = &mapping.mapper { if !mapper.is_empty() { - let header = &channel.header; - let channel_name = if mapping.match_as_ascii { deunicode(&header.name) } else { header.name.clone() }; - if mapping.match_as_ascii && log_enabled!(Level::Trace) { trace!("Decoded {} for matching to {}", &header.name, &channel_name); } let ref_chan = &mut channel; let templates = mapping.templates.as_ref(); for m in mapper { if let Some(script) = m.t_script.as_ref() { if let Some(filter) = &m.t_filter { - let provider = ValueProvider { pli: ref_chan }; + let provider = ValueProvider { pli: ref_chan, match_as_ascii: mapping.match_as_ascii }; if filter.filter(&provider) { matched = true; - let mut accessor = ValueAccessor { pli: ref_chan }; + let mut accessor = ValueAccessor { pli: ref_chan, virtual_items: vec![], match_as_ascii: mapping.match_as_ascii }; script.eval(&mut accessor, templates); + virtual_items.extend(accessor.virtual_items.into_iter().map(|(_, pli)| pli)); } } } } } } - (channel, matched) + (channel, virtual_items, matched) } fn map_channel_with_aliases(channel: PlaylistItem, mapping: &Mapping) -> Vec { - if mapping.create_alias { - let original = channel.clone(); - let (mut mapped_channel, matched) = map_channel(channel, mapping); - if matched { - mapped_channel.header.uuid = create_alias_uuid(original.header.get_uuid(), &mapping.id); - vec![original, mapped_channel] - } else { - vec![mapped_channel] - } - } else { - let (mapped_channel, _) = map_channel(channel, mapping); - vec![mapped_channel] - } + let (mapped_channel, mut virtual_items, _matched) = map_channel(channel, mapping); + let mut result = Vec::with_capacity(1 + virtual_items.len()); + + result.push(mapped_channel); + result.append(&mut virtual_items); + result } fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { @@ -264,7 +205,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &mut [PlaylistGroup]) { for counter in counter_list { for plg in &mut *playlist { for channel in &mut plg.channels { - let provider = ValueProvider { pli: channel }; + let provider = ValueProvider { pli: channel, match_as_ascii: mapping.match_as_ascii }; if counter.filter.filter(&provider) { let cntval = counter.value.fetch_add(1, core::sync::atomic::Ordering::AcqRel); let padded_cntval = if counter.padding > 0 { @@ -587,6 +528,9 @@ async fn process_playlist_for_target(app_config: &Arc, let (new_epg, mut new_playlist) = process_epg(&mut processed_fetched_playlists).await; step.tick("epg"); + process_favourites(&mut new_playlist, target.favourites.as_deref()); + + if new_playlist.is_empty() { step.stop(""); info!("Playlist is empty: {}", &target.name); @@ -618,6 +562,44 @@ async fn process_playlist_for_target(app_config: &Arc, } } +pub fn process_favourites(playlist: &mut Vec, favourites_cfg: Option<&[ConfigFavourites]>) { + if let Some(favourites) = favourites_cfg { + let mut fav_groups: HashMap> = HashMap::new(); + for pg in playlist.iter() { + for pli in &pg.channels { + // skip episodes, only containers or videos/live + if pli.header.item_type == PlaylistItemType::Series { + continue; + } + for fav in favourites { + if is_valid(pli, &fav.filter, false) { + let mut channel = pli.clone(); + channel.header.group.clone_from(&fav.group); + // Update UUID to be an alias of the original + channel.header.uuid = create_alias_uuid(&pli.header.uuid, &fav.group); + fav_groups + .entry(fav.group.clone()) + .or_default() + .push(channel); + } + } + } + } + + for (group_name, channels) in fav_groups { + if !channels.is_empty() { + let xtream_cluster = channels[0].header.xtream_cluster; + playlist.push(PlaylistGroup { + id: 0, + title: group_name, + channels, + xtream_cluster, + }); + } + } + } +} + async fn trakt_playlist(client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec, playlist: &mut Vec) -> bool { match process_trakt_categories_for_target(client, playlist, target).await { Ok(Some(trakt_categories)) => { diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index 56f8d6511..d8c1c0a75 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -1,5 +1,5 @@ use crate::utils::debug_if_enabled; -use log::{debug, error}; +use log::{debug, error, trace}; use path_clean::PathClean; use shared::error::str_to_io_error; use shared::utils::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, MAPPING_FILE, SOURCE_FILE, USER_FILE}; @@ -370,7 +370,7 @@ pub fn cleanup_unlisted_files_with_suffix( }; if delete && fs::remove_file(&path).is_ok() { - debug!("Deleted {:?}", path.display()); + trace!("Deleted {:?}", path.display()); } } } diff --git a/backend/src/utils/file/mapping_reader.rs b/backend/src/utils/file/mapping_reader.rs index 6403fd5e3..ca4282bef 100644 --- a/backend/src/utils/file/mapping_reader.rs +++ b/backend/src/utils/file/mapping_reader.rs @@ -48,10 +48,6 @@ fn merge_mappings(mappings: Vec) -> Vec { ..Default::default() }); - // Logic for match_as_ascii: true, if one of them is true - entry.match_as_ascii |= m.match_as_ascii; - entry.create_alias |= m.create_alias; - if let Some(mut mapper) = m.mapper.take() { entry.mapper.get_or_insert(vec![]).append(&mut mapper); } diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 0b3056f55..049b8292c 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -25,6 +25,7 @@ bytes = "1" ciborium = "0.2.2" hex = "0.4.3" lz4_flex = "0.12.0" +deunicode = "1.6.2" [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.82" diff --git a/shared/src/foundation/filter.rs b/shared/src/foundation/filter.rs index 7ece553fe..3e9056d17 100644 --- a/shared/src/foundation/filter.rs +++ b/shared/src/foundation/filter.rs @@ -47,21 +47,32 @@ pub fn set_field_value(pli: &mut PlaylistItem, field: ItemField, value: String) pub struct ValueProvider<'a> { pub pli: &'a PlaylistItem, + pub match_as_ascii: bool, } impl ValueProvider<'_> { pub fn get(&self, field: &str) -> Option> { - self.pli.header.get_field(field) + let val = self.pli.header.get_field(field)?; + if self.match_as_ascii { + return Some(Cow::Owned(deunicode::deunicode(&val))); + } + Some(val) } } pub struct ValueAccessor<'a> { pub pli: &'a mut PlaylistItem, + pub virtual_items: Vec<(String, PlaylistItem)>, + pub match_as_ascii: bool, } impl ValueAccessor<'_> { pub fn get(&self, field: &str) -> Option> { - self.pli.header.get_field(field) + let val = self.pli.header.get_field(field)?; + if self.match_as_ascii { + return Some(Cow::Owned(deunicode::deunicode(&val))); + } + Some(val) } pub fn set(&mut self, field: &str, value: &str) { @@ -792,6 +803,7 @@ mod tests { .filter(|&chan| { let provider = ValueProvider { pli: chan, + match_as_ascii: false, }; filter.filter(&provider) }) @@ -844,6 +856,7 @@ mod tests { .filter(|&chan| { let provider = ValueProvider { pli: chan, + match_as_ascii: false, }; filter.filter(&provider) }) @@ -905,6 +918,7 @@ mod tests { .filter(|&chan| { let provider = ValueProvider { pli: chan, + match_as_ascii: false, }; filter.filter(&provider) }) @@ -923,4 +937,29 @@ mod tests { } } } + + #[test] + fn test_filter_match_as_ascii() { + let flt = r#"Name ~ "Cinema""#; + match get_filter(flt, None) { + Ok(filter) => { + let chan = create_mock_pli("Cinéma", "Some Group"); + + // Without match_as_ascii (should fail) + let provider_fail = ValueProvider { + pli: &chan, + match_as_ascii: false, + }; + assert!(!filter.filter(&provider_fail)); + + // With match_as_ascii (should succeed) + let provider_success = ValueProvider { + pli: &chan, + match_as_ascii: true, + }; + assert!(filter.filter(&provider_success)); + } + Err(e) => panic!("{}", e), + } + } } diff --git a/shared/src/foundation/mapper.rs b/shared/src/foundation/mapper.rs index c5f34421b..2e6f0000c 100644 --- a/shared/src/foundation/mapper.rs +++ b/shared/src/foundation/mapper.rs @@ -40,7 +40,7 @@ block_expr = { "{" ~ statements ~ "}" } condition = { function_call | var_access | field_access } assignment = { (field_access | identifier) ~ "=" ~ expression } expression = { assignment | map_block | match_block | function_call | regex_expr | string_literal | number | var_access | field_access | null | block_expr } -function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "trim" | "print" | "number" | "first" | "template" | "replace" | "pad" | "format" } +function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "trim" | "print" | "number" | "first" | "template" | "replace" | "pad" | "format" | "add_favourite" } function_call = { function_name ~ "(" ~ (expression ~ ("," ~ expression)*)? ~ ")" } any_match = { "_" } match_case_key = { any_match | identifier } @@ -121,6 +121,7 @@ pub enum BuiltInFunction { Replace, Pad, Format, + AddFavourite, } impl FromStr for BuiltInFunction { @@ -140,6 +141,7 @@ impl FromStr for BuiltInFunction { "replace" => Ok(Self::Replace), "pad" => Ok(Self::Pad), "format" => Ok(Self::Format), + "add_favourite" => Ok(Self::AddFavourite), _ => create_tuliprox_error_result!(TuliproxErrorKind::Info, "Unknown function {s}"), } } @@ -160,6 +162,7 @@ impl Display for BuiltInFunction { Self::Replace => "replace", Self::Pad => "pad", Self::Format => "format", + Self::AddFavourite => "add_favourite", }.to_owned(); write!(f, "{str}") } @@ -699,7 +702,8 @@ impl<'a> MapperContext<'a> { match name { BuiltInFunction::ToNumber | BuiltInFunction::Template - | BuiltInFunction::First => { + | BuiltInFunction::First + | BuiltInFunction::AddFavourite => { if args.len() > 1 { return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Function accepts only one argument {:?}, {} given", name, args.len()); } @@ -1240,6 +1244,18 @@ impl Expression { Undefined } } + BuiltInFunction::AddFavourite => { + let group_name = extract_evaluated_arg_value!(evaluated_args, 0); + if let Some(group) = group_name { + if accessor.pli.header.item_type != crate::model::PlaylistItemType::Series { + let mut pli = accessor.pli.clone(); + pli.header.group.clone_from(group); + pli.header.uuid = crate::utils::create_alias_uuid(&accessor.pli.header.uuid, group); + accessor.virtual_items.push((group.clone(), pli)); + } + } + Undefined + } } } } @@ -1422,6 +1438,7 @@ mod tests { for pli in &mut channels { let mut accessor = ValueAccessor { pli, + virtual_items: vec![], }; mapper.eval(&mut accessor, None); println!("Result: {pli:?}"); @@ -1517,9 +1534,57 @@ mod tests { for pli in &mut channels { let mut accessor = ValueAccessor { pli, + virtual_items: vec![], }; mapper.eval(&mut accessor, None); println!("Result: {pli:?}"); } } + + #[test] + fn test_mapper_add_favourite() { + use crate::model::PlaylistItemType; + let dsl = r#" + add_favourite("My Favs"); + "#; + + let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); + + // Test with Video (should work) + let mut video = PlaylistItem { + header: PlaylistItemHeader { + name: "Movie 1".to_string(), + item_type: PlaylistItemType::Video, + ..Default::default() + } + }; + let mut accessor = ValueAccessor { pli: &mut video, virtual_items: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 1); + assert_eq!(accessor.virtual_items[0].1.header.group, "My Favs"); + + // Test with SeriesInfo (should work) + let mut series_info = PlaylistItem { + header: PlaylistItemHeader { + name: "Series 1".to_string(), + item_type: PlaylistItemType::SeriesInfo, + ..Default::default() + } + }; + let mut accessor = ValueAccessor { pli: &mut series_info, virtual_items: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 1); + + // Test with Series episode (should NOT work) + let mut episode = PlaylistItem { + header: PlaylistItemHeader { + name: "Episode 1".to_string(), + item_type: PlaylistItemType::Series, + ..Default::default() + } + }; + let mut accessor = ValueAccessor { pli: &mut episode, virtual_items: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 0); + } } diff --git a/shared/src/model/mapping.rs b/shared/src/model/mapping.rs index 13ae8ecab..9cfba0607 100644 --- a/shared/src/model/mapping.rs +++ b/shared/src/model/mapping.rs @@ -186,8 +186,6 @@ pub struct MappingDto { pub id: String, #[serde(default)] pub match_as_ascii: bool, - #[serde(default)] - pub create_alias: bool, pub mapper: Option>, pub counter: Option>, #[serde(skip_serializing, skip_deserializing)] diff --git a/shared/src/utils/hash_utils.rs b/shared/src/utils/hash_utils.rs index 8282e3b6a..4d1b25d95 100644 --- a/shared/src/utils/hash_utils.rs +++ b/shared/src/utils/hash_utils.rs @@ -127,4 +127,11 @@ pub fn parse_uuid_hex(s: &str) -> Option<[u8; 16]> { let decoded = hex::decode(buf).ok()?; decoded.try_into().ok() +} + +pub fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType { + let mut data = Vec::with_capacity(base_uuid.len() + mapping_id.len()); + data.extend_from_slice(base_uuid.as_ref()); + data.extend_from_slice(mapping_id.as_bytes()); + hash_bytes(&data) } \ No newline at end of file