added remove_duplicate target option

CHANGELOG.md

@@ -23,6 +23,7 @@ schedules:
 ```
 - Stats have now target information
 - Prevent simultaneous updates
+- Added target options `remove_duplicates` to remove entries with same `url`.
 
 # 2.0.10 (2024-12-03)
 - added Target Output Option `m3u_include_type_in_url`, default false. This adds `live`, `movie`, `series` to the url of the stream in reverse proxy mode.

README.md

@@ -390,6 +390,7 @@ Target options are:
 
 - `ignore_logo` logo attributes are ignored to avoid caching logo files on devices.
 - `share_live_streams` to share live stream connections  in reverse proxy mode.
+- `remove_duplicates` tries to remove duplicates by `url`.
 
 `strm` output has additional options
 - `underscore_whitespace` replaces all whitespaces with `_` in the path.

frontend/src/model/server-config.ts

@@ -57,6 +57,7 @@ export interface TargetConfig {
         m3u_include_type_in_url: boolean,
         m3u_mask_redirect_url: boolean,
         share_live_streams: boolean,
+        remove_duplicates: boolean,
     },
     sort: {
         match_as_ascii: boolean,

src/model/config.rs

@@ -314,6 +314,8 @@ pub struct ConfigTargetOptions {
     pub m3u_mask_redirect_url: bool,
     #[serde(default)]
     pub share_live_streams: bool,
+    #[serde(default)]
+    pub remove_duplicates: bool,
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]

src/processing/playlist_processor.rs

@@ -1,5 +1,6 @@
 extern crate unidecode;
 
+use crate::repository::storage::hash_string;
 use async_std::sync::Mutex;
 use core::cmp::Ordering;
 use std::cell::RefCell;
@@ -20,7 +21,7 @@ use crate::messaging::{send_message, MsgKind};
 use crate::model::config::{ConfigSortChannel, ConfigSortGroup, ConfigTarget, InputType,
                            ItemField, ProcessTargets, ProcessingOrder, SortOrder::{Asc, Desc}};
 use crate::model::mapping::{CounterModifier, Mapping, MappingValueProcessor};
-use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, XtreamCluster};
+use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor, PlaylistEntry, PlaylistGroup, PlaylistItem, UUIDType, XtreamCluster};
 use crate::model::stats::{InputStats, PlaylistStats, SourceStats, TargetStats};
 use crate::processing::affix_processor::apply_affixes;
 use crate::processing::playlist_watch::process_group_watch;
@@ -343,7 +344,7 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
         debug_if_enabled!("Source has {} groups", source_playlists.iter().map(|fpl| fpl.playlistgroups.len()).sum::<usize>());
         for target in &source.targets {
             if is_target_enabled(target, &user_targets) {
-                match process_playlist(&mut source_playlists, target, &cfg, &mut input_stats, &mut errors).await {
+                match process_playlist_for_target(&mut source_playlists, target, &cfg, &mut input_stats, &mut errors).await {
                     Ok(()) => {
                         target_stats.push(TargetStats::success(&target.name));
                     }
@@ -437,13 +438,23 @@ fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe {
     }
 }
 
+fn duplicate_hash(item: &PlaylistItem) -> UUIDType {
+    hash_string(&item.get_provider_url())
+}
 
-fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>) -> FetchedPlaylist<'a> {
+fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>, duplicates: &mut HashSet<UUIDType>) -> FetchedPlaylist<'a> {
     let mut new_fpl = FetchedPlaylist {
         input: fpl.input,
         playlistgroups: fpl.playlistgroups.clone(), // we need to clone, because of multiple target definitions, we cant change the initial playlist.
         epg: fpl.epg.clone(),
     };
+    if target.options.as_ref().is_some_and(|opt| opt.remove_duplicates) {
+        for group in &mut new_fpl.playlistgroups {
+            // `HashSet::insert`  returns true for first insert, otherweise false
+            group.channels.retain(|item| duplicates.insert(duplicate_hash(item)));
+        }
+    }
+
     for f in pipe {
         if let Some(groups) = f(&mut new_fpl.playlistgroups, target) {
             new_fpl.playlistgroups = groups;
@@ -474,17 +485,18 @@ fn flatten_groups(playlistgroups: Vec<PlaylistGroup>) -> Vec<PlaylistGroup> {
     sort_order
 }
 
-async fn process_playlist(playlists: &mut [FetchedPlaylist<'_>],
-                          target: &ConfigTarget,
-                          cfg: &Config,
-                          stats: &mut HashMap<u16, InputStats>,
-                          errors: &mut Vec<M3uFilterError>) -> Result<(), Vec<M3uFilterError>> {
+async fn process_playlist_for_target(playlists: &mut [FetchedPlaylist<'_>],
+                                     target: &ConfigTarget,
+                                     cfg: &Config,
+                                     stats: &mut HashMap<u16, InputStats>,
+                                     errors: &mut Vec<M3uFilterError>) -> Result<(), Vec<M3uFilterError>> {
     let pipe = get_processing_pipe(target);
     debug_if_enabled!("Processing order is {}", &target.processing_order);
 
+    let mut duplicates: HashSet<UUIDType> = HashSet::new();
     let mut processed_fetched_playlists: Vec<FetchedPlaylist> = vec![];
     for provider_fpl in playlists.iter_mut() {
-        let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl);
+        let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates);
         playlist_resolve_series(cfg, target, errors, &pipe, provider_fpl, &mut processed_fpl).await;
         playlist_resolve_vod(cfg, target, errors, &processed_fpl).await;
         // stats