Merge pull request #462 from knylbyte/feature/favourite-channels

create_alias mapping flag for favourite groups

README.md

@@ -1257,6 +1257,7 @@ This will replace all occurrences of `!delimiter!` and `!quality!` in the regexp
 Has the following top level entries:
 - `id` _mandatory_
 - `match_as_ascii` _optional_ default is `false`
+- `create_alias` _optional_ default is `false`
 - `mapper` _mandatory_
 - `counter` _optional_
 
@@ -1268,16 +1269,31 @@ If you have non ascii characters in you playlist and want to
 write regexp without considering chars like `é` and use `e` instead, set this option to `true`.
 [unidecode](https://crates.io/crates/unidecode) is used to convert the text.
 
-### 2.3.3 `mapper`
+### 2.3.3 `create_alias`
+Set this to `true` to keep the original channel and add a mapped copy whenever a mapper rule matches. The copy gets a derived UUID and follows the mapped fields (e.g. a new group), so a channel can live in multiple groups such as favourites.
+
+Example:
+```yaml
+mapping:
+  - id: favourites_news
+    match_as_ascii: true
+    create_alias: true
+    mapper:
+      - filter: 'Group ~ "(?i)news"'
+        script: |
+          @Group = "Favourites"
+```
+
+### 2.3.4 `mapper`
 Has the following top level entries:
 - `filter`
 - `script`
 
-#### 2.3.3.1 `filter`
+#### 2.3.4.1 `filter`
 The filter  is a string with a statement (@see filter statements).
 It is optional and allows you to filter the content.
 
-#### 2.3.3.2 `script`
+#### 2.3.4.2 `script`
 Script has a custom DSL syntax. 
 
 This Domain-Specific Language (DSL) supports simple scripting operations including variable assignment, 
@@ -1429,7 +1445,7 @@ mappings:
             @Caption = concat("!US_TNT_PREFIX!", " ", coast_quality)
             @Group = "!US_TNT_ENTERTAIN_GROUP!"
 ```
-### 2.3.4 counter
+### 2.3.5 counter
 
 Each mapping can have a list of counter.
 

backend/src/model/mapping.rs

@@ -30,6 +30,8 @@ pub struct Mapping {
     pub id: String,
     #[serde(default)]
     pub match_as_ascii: bool,
+    #[serde(default)]
+    pub create_alias: bool,
     pub mapper: Option<Vec<Mapper>>,
     pub counter: Option<Vec<MappingCounterDefinition>>,
     #[serde(skip_serializing, skip_deserializing)]
@@ -43,6 +45,7 @@ impl From<&MappingDto>  for Mapping {
         Self {
             id: dto.id.clone(),
             match_as_ascii: dto.match_as_ascii,
+            create_alias: dto.create_alias,
             mapper: dto.mapper.as_ref().map(|l| l.iter().map(Mapper::from).collect()),
             counter: dto.counter.clone(),
             t_counter: dto.t_counter.clone(),

backend/src/processing/processor/playlist.rs

@@ -33,7 +33,7 @@ use reqwest::Client;
 use shared::error::{get_errors_notify_message, notify_err, TuliproxError};
 use shared::foundation::filter::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider};
 use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster};
-use shared::utils::default_as_default;
+use shared::utils::{default_as_default, hash_bytes};
 use std::time::Instant;
 
 fn is_valid(pli: &PlaylistItem, filter: &Filter) -> bool {
@@ -170,7 +170,15 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt
     }
 }
 
-fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem {
+fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType {
+    let mut data = Vec::with_capacity(base_uuid.len() + mapping_id.len());
+    data.extend_from_slice(base_uuid);
+    data.extend_from_slice(mapping_id.as_bytes());
+    hash_bytes(&data)
+}
+
+fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> (PlaylistItem, bool) {
+    let mut matched = false;
     if let Some(mapper) = &mapping.mapper {
         if !mapper.is_empty() {
             let header = &channel.header;
@@ -183,6 +191,7 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem {
                     if let Some(filter) = &m.t_filter {
                         let provider = ValueProvider { pli: ref_chan };
                         if filter.filter(&provider) {
+                            matched = true;
                             let mut accessor = ValueAccessor { pli: ref_chan };
                             script.eval(&mut accessor, templates);
                         }
@@ -191,7 +200,23 @@ fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem {
             }
         }
     }
-    channel
+    (channel, matched)
+}
+
+fn map_channel_with_aliases(channel: PlaylistItem, mapping: &Mapping) -> Vec<PlaylistItem> {
+    if mapping.create_alias {
+        let original = channel.clone();
+        let (mut mapped_channel, matched) = map_channel(channel, mapping);
+        if matched {
+            mapped_channel.header.uuid = create_alias_uuid(original.header.get_uuid(), &mapping.id);
+            vec![original, mapped_channel]
+        } else {
+            vec![mapped_channel]
+        }
+    } else {
+        let (mapped_channel, _) = map_channel(channel, mapping);
+        vec![mapped_channel]
+    }
 }
 
 fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option<Vec<PlaylistGroup>> {
@@ -200,7 +225,7 @@ fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option
             let mut grp = playlist_group.clone();
             mappings.iter().filter(|&mapping| mapping.mapper.as_ref().is_some_and(|v| !v.is_empty()))
                 .for_each(|mapping|
-                    grp.channels = grp.channels.drain(..).map(|chan| map_channel(chan, mapping)).collect());
+                    grp.channels = grp.channels.drain(..).flat_map(|chan| map_channel_with_aliases(chan, mapping)).collect());
             grp
         }).collect();
 

backend/src/processing/processor/xtream_series.rs

@@ -11,7 +11,7 @@ use crate::repository::xtream_repository::{write_series_info_to_wal_file, xtream
 use crate::repository::IndexedDocumentReader;
 use shared::error::{notify_err, info_err};
 use crate::processing::processor::{handle_error, handle_error_and_return, create_resolve_options_function_for_xtream_target};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fs::File;
 use std::io::{BufWriter, Write};
 use std::sync::Arc;
@@ -53,6 +53,7 @@ fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &Ha
 async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc<reqwest::Client>, errors: &mut Vec<TuliproxError>,
                                       fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool {
     let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, fpl).await;
+    let mut fetched_in_run: HashSet<u32> = HashSet::new();
     // we cant write to the indexed-document directly because of the write lock and time-consuming operation.
     // All readers would be waiting for the lock and the app would be unresponsive.
     // We collect the content into a wal file and write it once we collected everything.
@@ -81,7 +82,7 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: Arc<reqwest::Clie
     let mut last_processed_series_info_count = 0;
     for pli in series_info_iter {
         let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids);
-        if should_update {
+        if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
             if let Some(content) = playlist_resolve_download_playlist_item(Arc::clone(&client), pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await {
                 let normalized_content = normalize_json_content(content);
                 handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, &normalized_content, &mut content_writer, &mut record_writer),

backend/src/processing/processor/xtream_vod.rs

@@ -10,7 +10,7 @@ use crate::processing::processor::{handle_error, handle_error_and_return, create
 use shared::utils::{get_u32_from_serde_value, get_u64_from_serde_value, get_string_from_serde_value};
 use crate::repository::xtream_repository::xtream_get_input_info;
 use serde_json::{from_str, Map, Value};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::{Write};
 use std::sync::Arc;
 use std::time::Instant;
@@ -77,6 +77,7 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc<reqwest::C
     else { return; };
 
     let mut processed_info_ids = read_processed_vod_info_ids(app_config, errors, fpl).await;
+    let mut fetched_in_run: HashSet<u32> = HashSet::new();
     let mut content_writer = utils::file_writer(&wal_content_file);
     let mut record_writer = utils::file_writer(&wal_record_file);
     let mut content_updated = false;
@@ -96,8 +97,8 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: Arc<reqwest::C
     let mut last_processed_vod_info_count = 0;
 
     for pli in  vod_info_iter {
-        let (should_update, _provider_id, _ts) = should_update_vod_info(pli, &processed_info_ids);
-        if should_update {
+        let (should_update, provider_id, _ts) = should_update_vod_info(pli, &processed_info_ids);
+        if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
             if let Some(content) = playlist_resolve_download_playlist_item(Arc::clone(&client), pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await {
                 let normalized_content = normalize_json_content(content);
                 if let Some((provider_id, info_record)) = extract_info_record_from_vod_info(&normalized_content) {

backend/src/utils/file/mapping_reader.rs

@@ -50,6 +50,7 @@ fn merge_mappings(mappings: Vec<MappingDto>) -> Vec<MappingDto> {
 
         // Logic for match_as_ascii: true, if one of them is true
         entry.match_as_ascii |= m.match_as_ascii;
+        entry.create_alias |= m.create_alias;
 
         if let Some(mut mapper) = m.mapper.take() {
             entry.mapper.get_or_insert(vec![]).append(&mut mapper);
@@ -136,4 +137,4 @@ pub fn read_mappings(mappings_file: &str, resolve_env: bool) -> Result<Option<Ma
         Some(dto) => Ok(Some(Mappings::from(&dto))),
         None => Ok(None),
     }
-}
+}

shared/src/model/mapping.rs

@@ -187,6 +187,8 @@ pub struct MappingDto {
     pub id: String,
     #[serde(default)]
     pub match_as_ascii: bool,
+    #[serde(default)]
+    pub create_alias: bool,
     pub mapper: Option<Vec<MapperDto>>,
     pub counter: Option<Vec<MappingCounterDefinition>>,
     #[serde(skip_serializing, skip_deserializing)]