From cd286de6647e3582e2d5780d34b97553f522e2d5 Mon Sep 17 00:00:00 2001 From: euzu Date: Mon, 6 Jan 2025 10:11:14 +0100 Subject: [PATCH] added `remove_duplicate` target option --- CHANGELOG.md | 1 + README.md | 1 + frontend/src/model/server-config.ts | 1 + src/model/config.rs | 2 ++ src/processing/playlist_processor.rs | 30 +++++++++++++++++++--------- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea86efd0b..e1703f65c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ schedules: ``` - Stats have now target information - Prevent simultaneous updates +- Added target options `remove_duplicates` to remove entries with same `url`. # 2.0.10 (2024-12-03) - added Target Output Option `m3u_include_type_in_url`, default false. This adds `live`, `movie`, `series` to the url of the stream in reverse proxy mode. diff --git a/README.md b/README.md index 16f30b5e4..3389a53c3 100644 --- a/README.md +++ b/README.md @@ -390,6 +390,7 @@ Target options are: - `ignore_logo` logo attributes are ignored to avoid caching logo files on devices. - `share_live_streams` to share live stream connections in reverse proxy mode. +- `remove_duplicates` tries to remove duplicates by `url`. `strm` output has additional options - `underscore_whitespace` replaces all whitespaces with `_` in the path. diff --git a/frontend/src/model/server-config.ts b/frontend/src/model/server-config.ts index 960971376..8717cb9a8 100644 --- a/frontend/src/model/server-config.ts +++ b/frontend/src/model/server-config.ts @@ -57,6 +57,7 @@ export interface TargetConfig { m3u_include_type_in_url: boolean, m3u_mask_redirect_url: boolean, share_live_streams: boolean, + remove_duplicates: boolean, }, sort: { match_as_ascii: boolean, diff --git a/src/model/config.rs b/src/model/config.rs index ec1683ded..ed32fd145 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -314,6 +314,8 @@ pub struct ConfigTargetOptions { pub m3u_mask_redirect_url: bool, #[serde(default)] pub share_live_streams: bool, + #[serde(default)] + pub remove_duplicates: bool, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] diff --git a/src/processing/playlist_processor.rs b/src/processing/playlist_processor.rs index 12cd92989..94841962d 100644 --- a/src/processing/playlist_processor.rs +++ b/src/processing/playlist_processor.rs @@ -1,5 +1,6 @@ extern crate unidecode; +use crate::repository::storage::hash_string; use async_std::sync::Mutex; use core::cmp::Ordering; use std::cell::RefCell; @@ -20,7 +21,7 @@ use crate::messaging::{send_message, MsgKind}; use crate::model::config::{ConfigSortChannel, ConfigSortGroup, ConfigTarget, InputType, ItemField, ProcessTargets, ProcessingOrder, SortOrder::{Asc, Desc}}; use crate::model::mapping::{CounterModifier, Mapping, MappingValueProcessor}; -use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, XtreamCluster}; +use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor, PlaylistEntry, PlaylistGroup, PlaylistItem, UUIDType, XtreamCluster}; use crate::model::stats::{InputStats, PlaylistStats, SourceStats, TargetStats}; use crate::processing::affix_processor::apply_affixes; use crate::processing::playlist_watch::process_group_watch; @@ -343,7 +344,7 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

()); for target in &source.targets { if is_target_enabled(target, &user_targets) { - match process_playlist(&mut source_playlists, target, &cfg, &mut input_stats, &mut errors).await { + match process_playlist_for_target(&mut source_playlists, target, &cfg, &mut input_stats, &mut errors).await { Ok(()) => { target_stats.push(TargetStats::success(&target.name)); } @@ -437,13 +438,23 @@ fn get_processing_pipe(target: &ConfigTarget) -> ProcessingPipe { } } +fn duplicate_hash(item: &PlaylistItem) -> UUIDType { + hash_string(&item.get_provider_url()) +} -fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>) -> FetchedPlaylist<'a> { +fn execute_pipe<'a>(target: &ConfigTarget, pipe: &ProcessingPipe, fpl: &FetchedPlaylist<'a>, duplicates: &mut HashSet) -> FetchedPlaylist<'a> { let mut new_fpl = FetchedPlaylist { input: fpl.input, playlistgroups: fpl.playlistgroups.clone(), // we need to clone, because of multiple target definitions, we cant change the initial playlist. epg: fpl.epg.clone(), }; + if target.options.as_ref().is_some_and(|opt| opt.remove_duplicates) { + for group in &mut new_fpl.playlistgroups { + // `HashSet::insert` returns true for first insert, otherweise false + group.channels.retain(|item| duplicates.insert(duplicate_hash(item))); + } + } + for f in pipe { if let Some(groups) = f(&mut new_fpl.playlistgroups, target) { new_fpl.playlistgroups = groups; @@ -474,17 +485,18 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { sort_order } -async fn process_playlist(playlists: &mut [FetchedPlaylist<'_>], - target: &ConfigTarget, - cfg: &Config, - stats: &mut HashMap, - errors: &mut Vec) -> Result<(), Vec> { +async fn process_playlist_for_target(playlists: &mut [FetchedPlaylist<'_>], + target: &ConfigTarget, + cfg: &Config, + stats: &mut HashMap, + errors: &mut Vec) -> Result<(), Vec> { let pipe = get_processing_pipe(target); debug_if_enabled!("Processing order is {}", &target.processing_order); + let mut duplicates: HashSet = HashSet::new(); let mut processed_fetched_playlists: Vec = vec![]; for provider_fpl in playlists.iter_mut() { - let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl); + let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); playlist_resolve_series(cfg, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; playlist_resolve_vod(cfg, target, errors, &processed_fpl).await; // stats