From 5a10efe82e94ebb5bd5b118deca5b367663eebf7 Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Sun, 30 Nov 2025 18:35:01 +0100 Subject: [PATCH 1/4] feat: added option to create an alias during mapping --- backend/src/model/mapping.rs | 3 ++ backend/src/processing/processor/playlist.rs | 33 +++++++++++++++++--- backend/src/utils/file/mapping_reader.rs | 3 +- shared/src/model/mapping.rs | 2 ++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/backend/src/model/mapping.rs b/backend/src/model/mapping.rs index dd23f3af1..b8d0d6241 100644 --- a/backend/src/model/mapping.rs +++ b/backend/src/model/mapping.rs @@ -30,6 +30,8 @@ pub struct Mapping { pub id: String, #[serde(default)] pub match_as_ascii: bool, + #[serde(default)] + pub create_alias: bool, pub mapper: Option>, pub counter: Option>, #[serde(skip_serializing, skip_deserializing)] @@ -43,6 +45,7 @@ impl From<&MappingDto> for Mapping { Self { id: dto.id.clone(), match_as_ascii: dto.match_as_ascii, + create_alias: dto.create_alias, mapper: dto.mapper.as_ref().map(|l| l.iter().map(Mapper::from).collect()), counter: dto.counter.clone(), t_counter: dto.t_counter.clone(), diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index ec2f5ee1e..b168343f8 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -33,7 +33,7 @@ use reqwest::Client; use shared::error::{get_errors_notify_message, notify_err, TuliproxError}; use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider}; use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster}; -use shared::utils::default_as_default; +use shared::utils::{default_as_default, hash_bytes}; use std::time::Instant; fn is_valid(pli: &PlaylistItem, filter: &Filter) -> bool { @@ -170,7 +170,15 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt } } -fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem { +fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType { + let mut data = Vec::with_capacity(base_uuid.len() + mapping_id.len()); + data.extend_from_slice(base_uuid); + data.extend_from_slice(mapping_id.as_bytes()); + hash_bytes(&data) +} + +fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, bool) { + let mut matched = false; if let Some(mapper) = &mapping.mapper { if !mapper.is_empty() { let header = &channel.header; @@ -183,6 +191,7 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem { if let Some(filter) = &m.t_filter { let provider = ValueProvider { pli: ref_chan }; if filter.filter(&provider) { + matched = true; let mut accessor = ValueAccessor { pli: ref_chan }; script.eval(&mut accessor, templates); } @@ -191,7 +200,23 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem { } } } - channel + (channel, matched) +} + +fn map_channel_with_aliases(channel: PlaylistItem, mapping: &Mapping) -> Vec { + if mapping.create_alias { + let original = channel.clone(); + let (mut mapped_channel, matched) = map_channel(channel, mapping); + if matched { + mapped_channel.header.uuid = create_alias_uuid(original.header.get_uuid(), &mapping.id); + vec![original, mapped_channel] + } else { + vec![mapped_channel] + } + } else { + let (mapped_channel, _) = map_channel(channel, mapping); + vec![mapped_channel] + } } fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { @@ -200,7 +225,7 @@ fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option let mut grp = playlist_group.clone(); mappings.iter().filter(|&mapping| mapping.mapper.as_ref().is_some_and(|v| !v.is_empty())) .for_each(|mapping| - grp.channels = grp.channels.drain(..).map(|chan| map_channel(chan, mapping)).collect()); + grp.channels = grp.channels.drain(..).flat_map(|chan| map_channel_with_aliases(chan, mapping)).collect()); grp }).collect(); diff --git a/backend/src/utils/file/mapping_reader.rs b/backend/src/utils/file/mapping_reader.rs index 40c839ccf..6403fd5e3 100644 --- a/backend/src/utils/file/mapping_reader.rs +++ b/backend/src/utils/file/mapping_reader.rs @@ -50,6 +50,7 @@ fn merge_mappings(mappings: Vec) -> Vec { // Logic for match_as_ascii: true, if one of them is true entry.match_as_ascii |= m.match_as_ascii; + entry.create_alias |= m.create_alias; if let Some(mut mapper) = m.mapper.take() { entry.mapper.get_or_insert(vec![]).append(&mut mapper); @@ -136,4 +137,4 @@ pub fn read_mappings(mappings_file: &str, resolve_env: bool) -> Result Ok(Some(Mappings::from(&dto))), None => Ok(None), } -} \ No newline at end of file +} diff --git a/shared/src/model/mapping.rs b/shared/src/model/mapping.rs index 26345c8fc..b7ec5270d 100644 --- a/shared/src/model/mapping.rs +++ b/shared/src/model/mapping.rs @@ -187,6 +187,8 @@ pub struct MappingDto { pub id: String, #[serde(default)] pub match_as_ascii: bool, + #[serde(default)] + pub create_alias: bool, pub mapper: Option>, pub counter: Option>, #[serde(skip_serializing, skip_deserializing)] From df26c72d531e42fef297281254ae8b464dfa8aa8 Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Sun, 30 Nov 2025 19:16:57 +0100 Subject: [PATCH 2/4] feat: added option to create an alias during mapping --- README.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7c58695b7..3071c6fc1 100644 --- a/README.md +++ b/README.md @@ -1257,6 +1257,7 @@ This will replace all occurrences of `!delimiter!` and `!quality!` in the regexp Has the following top level entries: - `id` _mandatory_ - `match_as_ascii` _optional_ default is `false` +- `create_alias` _optional_ default is `false` - `mapper` _mandatory_ - `counter` _optional_ @@ -1268,16 +1269,31 @@ If you have non ascii characters in you playlist and want to write regexp without considering chars like `é` and use `e` instead, set this option to `true`. [unidecode](https://crates.io/crates/unidecode) is used to convert the text. -### 2.3.3 `mapper` +### 2.3.3 `create_alias` +Set this to `true` to keep the original channel and add a mapped copy whenever a mapper rule matches. The copy gets a derived UUID and follows the mapped fields (e.g. a new group), so a channel can live in multiple groups such as favourites. + +Example: +```yaml +mapping: + - id: favourites_news + match_as_ascii: true + create_alias: true + mapper: + - filter: 'Group ~ "(?i)news"' + script: | + @Group = "Favourites" +``` + +### 2.3.4 `mapper` Has the following top level entries: - `filter` - `script` -#### 2.3.3.1 `filter` +#### 2.3.4.1 `filter` The filter is a string with a statement (@see filter statements). It is optional and allows you to filter the content. -#### 2.3.3.2 `script` +#### 2.3.4.2 `script` Script has a custom DSL syntax. This Domain-Specific Language (DSL) supports simple scripting operations including variable assignment, @@ -1429,7 +1445,7 @@ mappings: @Caption = concat("!US_TNT_PREFIX!", " ", coast_quality) @Group = "!US_TNT_ENTERTAIN_GROUP!" ``` -### 2.3.4 counter +### 2.3.5 counter Each mapping can have a list of counter. From d68f6df31158d67dcac18652aa8c177a2469ce87 Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Sun, 30 Nov 2025 19:56:51 +0100 Subject: [PATCH 3/4] feat: prevent duplicate xtream info fetches for mapping aliases --- backend/src/processing/processor/playlist.rs | 2 ++ backend/src/processing/processor/xtream.rs | 3 +++ shared/src/model/playlist.rs | 2 ++ 3 files changed, 7 insertions(+) diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index b168343f8..c2a0fcb50 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -206,9 +206,11 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, b fn map_channel_with_aliases(channel: PlaylistItem, mapping: &Mapping) -> Vec { if mapping.create_alias { let original = channel.clone(); + let base_uuid = *original.header.get_uuid(); let (mut mapped_channel, matched) = map_channel(channel, mapping); if matched { mapped_channel.header.uuid = create_alias_uuid(original.header.get_uuid(), &mapping.id); + mapped_channel.header.alias_of = Some(base_uuid); vec![original, mapped_channel] } else { vec![mapped_channel] diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index f7591d133..fb10203c1 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -82,6 +82,9 @@ pub(in crate::processing) fn create_resolve_info_wal_files(cfg: &Config, input: pub(in crate::processing) fn should_update_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap, field: &str) -> (bool, u32, u64) { let Some(provider_id) = pli.header.get_provider_id() else { return (false, 0, 0) }; let last_modified = pli.header.get_additional_property_as_u64(field); + if pli.header.alias_of.is_some() { + return (false, provider_id, last_modified.unwrap_or(0)); + } let old_timestamp = processed_provider_ids.get(&provider_id); (old_timestamp.is_none() || last_modified.is_none() diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index 937ebcf85..e818e9be5 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -150,6 +150,8 @@ pub trait PlaylistEntry: Send + Sync { pub struct PlaylistItemHeader { #[serde(skip)] pub uuid: UUIDType, // calculated + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alias_of: Option, pub id: String, // provider id pub virtual_id: u32, // virtual id pub name: String, From adb9e70807cf1f587d45362c821a4f6a08604ade Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:29:00 +0100 Subject: [PATCH 4/4] feat: prevent duplicate xtream info fetches for mapping aliases --- backend/src/processing/processor/playlist.rs | 2 -- backend/src/processing/processor/xtream.rs | 3 --- backend/src/processing/processor/xtream_series.rs | 5 +++-- backend/src/processing/processor/xtream_vod.rs | 7 ++++--- shared/src/model/playlist.rs | 2 -- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index c2a0fcb50..b168343f8 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -206,11 +206,9 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, b fn map_channel_with_aliases(channel: PlaylistItem, mapping: &Mapping) -> Vec { if mapping.create_alias { let original = channel.clone(); - let base_uuid = *original.header.get_uuid(); let (mut mapped_channel, matched) = map_channel(channel, mapping); if matched { mapped_channel.header.uuid = create_alias_uuid(original.header.get_uuid(), &mapping.id); - mapped_channel.header.alias_of = Some(base_uuid); vec![original, mapped_channel] } else { vec![mapped_channel] diff --git a/backend/src/processing/processor/xtream.rs b/backend/src/processing/processor/xtream.rs index fb10203c1..f7591d133 100644 --- a/backend/src/processing/processor/xtream.rs +++ b/backend/src/processing/processor/xtream.rs @@ -82,9 +82,6 @@ pub(in crate::processing) fn create_resolve_info_wal_files(cfg: &Config, input: pub(in crate::processing) fn should_update_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap, field: &str) -> (bool, u32, u64) { let Some(provider_id) = pli.header.get_provider_id() else { return (false, 0, 0) }; let last_modified = pli.header.get_additional_property_as_u64(field); - if pli.header.alias_of.is_some() { - return (false, provider_id, last_modified.unwrap_or(0)); - } let old_timestamp = processed_provider_ids.get(&provider_id); (old_timestamp.is_none() || last_modified.is_none() diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 2fe3cbcfe..9389cb6e8 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -11,7 +11,7 @@ use crate::repository::xtream_repository::{write_series_info_to_wal_file, xtream use crate::repository::IndexedDocumentReader; use shared::error::{notify_err, info_err}; use crate::processing::processor::{handle_error, handle_error_and_return, create_resolve_options_function_for_xtream_target}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{BufWriter, Write}; use std::sync::Arc; @@ -53,6 +53,7 @@ fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &Ha async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc, errors: &mut Vec, fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, fpl).await; + let mut fetched_in_run: HashSet = HashSet::new(); // we cant write to the indexed-document directly because of the write lock and time-consuming operation. // All readers would be waiting for the lock and the app would be unresponsive. // We collect the content into a wal file and write it once we collected everything. @@ -81,7 +82,7 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc = HashSet::new(); let mut content_writer = utils::file_writer(&wal_content_file); let mut record_writer = utils::file_writer(&wal_record_file); let mut content_updated = false; @@ -96,8 +97,8 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc, pub id: String, // provider id pub virtual_id: u32, // virtual id pub name: String,