Feature/failover (#924)

* feature stream failover

backend/processing/src/processor/channel_group/assemble.rs

@@ -0,0 +1,544 @@
+use crate::processor::match_key::{extract_normalized_match_key, MatchKeyNormalizer};
+use indexmap::IndexMap;
+use shared::{
+    model::{
+        ChannelGroupOverridesDto, ChannelGroupPolicy, ChannelGroupsConfig, ChannelMemberRefDto, EpgConflictMode,
+        LiveStreamProperties, PlaylistGroup, PlaylistItem, StreamProperties, UUIDType, XtreamCluster,
+        MIN_GROUP_MEMBERS,
+    },
+    utils::{hash_string, token_quality, Internable},
+};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
+use tuliprox_repository::{ChannelGroupOrigin, PendingChannelGroup};
+
+pub(in crate::processor) fn strip_quality_words(value: &str) -> String {
+    let words: Vec<&str> = value.split_whitespace().collect();
+    let mut kept = Vec::with_capacity(words.len());
+    for word in words {
+        let core = word.trim_matches(|c: char| !c.is_alphanumeric());
+        if !core.is_empty() && token_quality(core).is_some() {
+            continue;
+        }
+        kept.push(word);
+    }
+    if kept.is_empty() {
+        value.to_string()
+    } else {
+        kept.join(" ")
+    }
+}
+
+fn is_eligible(item: &PlaylistItem) -> bool {
+    item.header.xtream_cluster == XtreamCluster::Live && !item.header.item_type.is_local()
+}
+
+fn item_member_ref(item: &PlaylistItem) -> Option<ChannelMemberRefDto> {
+    if !is_eligible(item) {
+        return None;
+    }
+    let stream_id = item.header.get_input_stream_id()?;
+    if stream_id.is_empty() {
+        return None;
+    }
+    Some(ChannelMemberRefDto { input: item.header.input_name.to_string(), stream_id: stream_id.to_string() })
+}
+
+struct GroupCandidate {
+    key: String,
+    origin: ChannelGroupOrigin,
+    policy: ChannelGroupPolicy,
+    positions: Vec<(usize, usize)>,
+    orphaned: Vec<ChannelMemberRefDto>,
+    epg_split: bool,
+    name_override: Option<String>,
+    logo_override: Option<String>,
+    epg_override: Option<String>,
+    playlist_group_override: Option<String>,
+}
+
+struct AutoBucket {
+    positions: Vec<(usize, usize)>,
+    has_dsl_forced: bool,
+    epg_split: bool,
+}
+
+struct GroupSlot {
+    id: u32,
+    title: Arc<str>,
+    xtream_cluster: XtreamCluster,
+    had_channels: bool,
+    channels: Vec<Option<PlaylistItem>>,
+}
+
+fn build_member_index(playlist: &[PlaylistGroup]) -> HashMap<ChannelMemberRefDto, (usize, usize)> {
+    let mut member_pos_map = HashMap::new();
+    for (group_idx, group) in playlist.iter().enumerate() {
+        if group.xtream_cluster != XtreamCluster::Live {
+            continue;
+        }
+        for (channel_idx, channel) in group.channels.iter().enumerate() {
+            if let Some(mref) = item_member_ref(channel) {
+                member_pos_map.entry(mref).or_insert((group_idx, channel_idx));
+            }
+        }
+    }
+    member_pos_map
+}
+
+fn resolve_manual_groups(
+    config: &ChannelGroupsConfig,
+    overrides: Option<&ChannelGroupOverridesDto>,
+    member_pos_map: &HashMap<ChannelMemberRefDto, (usize, usize)>,
+    taken_positions: &mut HashSet<(usize, usize)>,
+    candidates: &mut Vec<GroupCandidate>,
+) {
+    let Some(overrides) = overrides else {
+        return;
+    };
+    for mg in &overrides.groups {
+        let mut matched_positions = Vec::new();
+        let mut orphaned = Vec::new();
+        for mref in &mg.members {
+            if let Some(&pos) = member_pos_map.get(mref) {
+                if !taken_positions.contains(&pos) {
+                    taken_positions.insert(pos);
+                    matched_positions.push(pos);
+                    continue;
+                }
+            }
+            log::warn!(
+                "Manual channel group '{}' member '{}:{}' not found or already taken",
+                mg.id,
+                mref.input,
+                mref.stream_id
+            );
+            orphaned.push(mref.clone());
+        }
+
+        if matched_positions.len() < MIN_GROUP_MEMBERS {
+            continue;
+        }
+
+        candidates.push(GroupCandidate {
+            key: mg.id.clone(),
+            origin: ChannelGroupOrigin::Manual,
+            policy: mg.policy.unwrap_or(config.policy),
+            positions: matched_positions,
+            orphaned,
+            epg_split: false,
+            name_override: Some(mg.name.clone()),
+            logo_override: mg.logo.clone(),
+            epg_override: mg.epg_channel_id.clone(),
+            playlist_group_override: mg.playlist_group.clone(),
+        });
+    }
+}
+
+fn collect_auto_buckets(
+    config: &ChannelGroupsConfig,
+    playlist: &[PlaylistGroup],
+    excluded_set: &HashSet<&ChannelMemberRefDto>,
+    taken_positions: &HashSet<(usize, usize)>,
+) -> IndexMap<String, AutoBucket> {
+    let normalizer = MatchKeyNormalizer::new(&config.match_normalize, config.match_as_ascii);
+    let mut auto_buckets: IndexMap<String, AutoBucket> = IndexMap::new();
+
+    for (group_idx, group) in playlist.iter().enumerate() {
+        if group.xtream_cluster != XtreamCluster::Live {
+            continue;
+        }
+        for (channel_idx, channel) in group.channels.iter().enumerate() {
+            let pos = (group_idx, channel_idx);
+            if taken_positions.contains(&pos) {
+                continue;
+            }
+            let Some(mref) = item_member_ref(channel) else {
+                continue;
+            };
+            if excluded_set.contains(&mref) {
+                continue;
+            }
+
+            let mut dsl_forced = false;
+            let key = match channel.header.channel_group.as_deref() {
+                Some("") => continue,
+                Some(forced) => {
+                    dsl_forced = true;
+                    normalizer.key(forced)
+                }
+                None => extract_normalized_match_key(
+                    &channel.header,
+                    config.match_by,
+                    config.match_as_ascii,
+                    Some(&normalizer),
+                ),
+            };
+
+            if key.trim().is_empty() {
+                continue;
+            }
+
+            let bucket = auto_buckets.entry(key).or_insert_with(|| AutoBucket {
+                positions: Vec::new(),
+                has_dsl_forced: false,
+                epg_split: false,
+            });
+            bucket.positions.push(pos);
+            if dsl_forced {
+                bucket.has_dsl_forced = true;
+            }
+        }
+    }
+    auto_buckets
+}
+
+fn split_epg_buckets(
+    playlist: &[PlaylistGroup],
+    auto_buckets: IndexMap<String, AutoBucket>,
+) -> IndexMap<String, AutoBucket> {
+    let mut split_buckets: IndexMap<String, AutoBucket> = IndexMap::new();
+    for (key, bucket) in auto_buckets {
+        if bucket.has_dsl_forced {
+            split_buckets.insert(key, bucket);
+            continue;
+        }
+
+        let mut epg_groups: IndexMap<String, Vec<(usize, usize)>> = IndexMap::new();
+        for &pos in &bucket.positions {
+            let ch = &playlist[pos.0].channels[pos.1];
+            if let Some(epg) = ch.header.epg_channel_id.as_deref() {
+                let trimmed = epg.trim().to_lowercase();
+                if !trimmed.is_empty() {
+                    epg_groups.entry(trimmed).or_default().push(pos);
+                }
+            }
+        }
+
+        if epg_groups.len() >= 2 {
+            for (epg_id, sub_positions) in epg_groups {
+                let sub_key = format!("{key}|{epg_id}");
+                split_buckets
+                    .insert(sub_key, AutoBucket { positions: sub_positions, has_dsl_forced: false, epg_split: true });
+            }
+        } else {
+            split_buckets.insert(key, bucket);
+        }
+    }
+    split_buckets
+}
+
+fn apply_auto_override(
+    bucket: &mut AutoBucket,
+    ao: &shared::model::AutoGroupOverrideDto,
+    playlist: &[PlaylistGroup],
+    member_pos_map: &HashMap<ChannelMemberRefDto, (usize, usize)>,
+    taken_positions: &HashSet<(usize, usize)>,
+    excluded_set: &HashSet<&ChannelMemberRefDto>,
+) {
+    if !ao.remove.is_empty() {
+        bucket.positions.retain(|&pos| {
+            let ch = &playlist[pos.0].channels[pos.1];
+            !ao.remove.iter().any(|rem| {
+                ch.header.input_name.as_ref() == rem.input.as_str()
+                    && ch.header.get_input_stream_id().as_deref() == Some(rem.stream_id.as_str())
+            })
+        });
+    }
+
+    for add_ref in &ao.add {
+        if let Some(&pos) = member_pos_map.get(add_ref) {
+            if !taken_positions.contains(&pos) && !excluded_set.contains(add_ref) && !bucket.positions.contains(&pos) {
+                bucket.positions.push(pos);
+            }
+        }
+    }
+
+    if !ao.order.is_empty() {
+        let mut ordered = Vec::with_capacity(bucket.positions.len());
+        let mut used = HashSet::new();
+
+        for ord_ref in &ao.order {
+            if let Some(&pos) = bucket.positions.iter().find(|&&pos| {
+                let ch = &playlist[pos.0].channels[pos.1];
+                ch.header.input_name.as_ref() == ord_ref.input.as_str()
+                    && ch.header.get_input_stream_id().as_deref() == Some(ord_ref.stream_id.as_str())
+            }) {
+                if used.insert(pos) {
+                    ordered.push(pos);
+                }
+            }
+        }
+        for &pos in &bucket.positions {
+            if !used.contains(&pos) {
+                ordered.push(pos);
+            }
+        }
+        bucket.positions = ordered;
+    }
+}
+
+fn build_group_metadata(
+    candidate: &GroupCandidate,
+    rep_header: &shared::model::PlaylistItemHeader,
+    slots: &[GroupSlot],
+    config: &ChannelGroupsConfig,
+    group_uuid: UUIDType,
+) -> shared::model::PlaylistItemHeader {
+    let mut group_header = rep_header.clone();
+    group_header.uuid = group_uuid;
+    group_header.channel_group = None;
+    group_header.channel_group_uuid = Some(group_uuid);
+
+    if let Some(name) = &candidate.name_override {
+        group_header.name = name.as_str().intern();
+        group_header.title = name.as_str().intern();
+    } else if config.name_strip_quality {
+        group_header.name = strip_quality_words(&rep_header.name).intern();
+        group_header.title = strip_quality_words(&rep_header.title).intern();
+    }
+
+    if let Some(logo) = &candidate.logo_override {
+        group_header.logo = logo.as_str().intern();
+    } else if group_header.logo.is_empty() {
+        for &pos in &candidate.positions {
+            if let Some(item) = slots.get(pos.0).and_then(|g| g.channels.get(pos.1)).and_then(Option::as_ref) {
+                if !item.header.logo.is_empty() {
+                    group_header.logo = item.header.logo.clone();
+                    break;
+                }
+            }
+        }
+    }
+
+    if let Some(epg) = &candidate.epg_override {
+        group_header.epg_channel_id = Some(epg.as_str().intern());
+    } else if group_header.epg_channel_id.as_deref().is_none_or(|s| s.trim().is_empty()) {
+        for &pos in &candidate.positions {
+            if let Some(item) = slots.get(pos.0).and_then(|g| g.channels.get(pos.1)).and_then(Option::as_ref) {
+                if let Some(epg) = &item.header.epg_channel_id {
+                    if !epg.trim().is_empty() {
+                        group_header.epg_channel_id = Some(epg.clone());
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    apply_archive_metadata(&mut group_header, candidate, slots);
+    group_header
+}
+
+fn apply_archive_metadata(
+    group_header: &mut shared::model::PlaylistItemHeader,
+    candidate: &GroupCandidate,
+    slots: &[GroupSlot],
+) {
+    let mut has_archive = false;
+    let mut max_archive_duration: Option<i32> = None;
+    for &pos in &candidate.positions {
+        if let Some(item) = slots.get(pos.0).and_then(|g| g.channels.get(pos.1)).and_then(Option::as_ref) {
+            if let Some(StreamProperties::Live(live)) = &item.header.additional_properties {
+                if live.tv_archive.unwrap_or(0) > 0 {
+                    has_archive = true;
+                    if let Some(dur) = live.tv_archive_duration {
+                        max_archive_duration = Some(max_archive_duration.map_or(dur, |m| m.max(dur)));
+                    }
+                }
+            }
+        }
+    }
+
+    if has_archive {
+        match &mut group_header.additional_properties {
+            Some(StreamProperties::Live(live)) => {
+                live.tv_archive = Some(1);
+                live.tv_archive_duration = max_archive_duration;
+            }
+            _ => {
+                group_header.additional_properties = Some(StreamProperties::Live(Box::new(LiveStreamProperties {
+                    name: group_header.name.clone(),
+                    category_id: group_header.category_id,
+                    stream_id: 0,
+                    stream_icon: group_header.logo.clone(),
+                    direct_source: "".intern(),
+                    custom_sid: None,
+                    added: None,
+                    stream_type: None,
+                    epg_channel_id: group_header.epg_channel_id.clone(),
+                    tv_archive: Some(1),
+                    tv_archive_duration: max_archive_duration,
+                    is_adult: 0,
+                    ..Default::default()
+                })));
+            }
+        }
+    }
+}
+
+fn collect_group_candidates(
+    config: &ChannelGroupsConfig,
+    overrides: Option<&ChannelGroupOverridesDto>,
+    playlist: &[PlaylistGroup],
+) -> Vec<GroupCandidate> {
+    let member_pos_map = build_member_index(playlist);
+    let mut taken_positions = HashSet::new();
+    let mut candidates = Vec::new();
+
+    resolve_manual_groups(config, overrides, &member_pos_map, &mut taken_positions, &mut candidates);
+
+    let excluded_set: HashSet<&ChannelMemberRefDto> =
+        overrides.map_or_else(HashSet::new, |o| o.excluded.iter().collect());
+
+    let mut auto_buckets = collect_auto_buckets(config, playlist, &excluded_set, &taken_positions);
+
+    if config.epg_conflict == EpgConflictMode::Split {
+        auto_buckets = split_epg_buckets(playlist, auto_buckets);
+    }
+
+    for (key, mut bucket) in auto_buckets {
+        let auto_override = overrides.and_then(|o| o.auto_overrides.iter().find(|ao| ao.auto_key == key));
+        let origin = if auto_override.is_some() { ChannelGroupOrigin::AutoEdited } else { ChannelGroupOrigin::Auto };
+
+        if let Some(ao) = auto_override {
+            apply_auto_override(&mut bucket, ao, playlist, &member_pos_map, &taken_positions, &excluded_set);
+        }
+
+        if bucket.positions.len() < MIN_GROUP_MEMBERS {
+            continue;
+        }
+
+        for &pos in &bucket.positions {
+            taken_positions.insert(pos);
+        }
+
+        candidates.push(GroupCandidate {
+            key: key.clone(),
+            origin,
+            policy: auto_override.and_then(|ao| ao.policy).unwrap_or(config.policy),
+            positions: bucket.positions,
+            orphaned: Vec::new(),
+            epg_split: bucket.epg_split,
+            name_override: auto_override.and_then(|ao| ao.name.clone()),
+            logo_override: auto_override.and_then(|ao| ao.logo.clone()),
+            epg_override: auto_override.and_then(|ao| ao.epg_channel_id.clone()),
+            playlist_group_override: auto_override.and_then(|ao| ao.playlist_group.clone()),
+        });
+    }
+
+    candidates
+}
+
+fn rebuild_playlist_from_slots(
+    slots: Vec<GroupSlot>,
+    deferred_placements: Vec<(String, PlaylistItem)>,
+    playlist: &mut Vec<PlaylistGroup>,
+) {
+    for slot in slots {
+        let channels: Vec<PlaylistItem> = slot.channels.into_iter().flatten().collect();
+        if channels.is_empty() && slot.had_channels {
+            continue;
+        }
+        playlist.push(PlaylistGroup { id: slot.id, title: slot.title, channels, xtream_cluster: slot.xtream_cluster });
+    }
+
+    for (pg_override, mut item) in deferred_placements {
+        if let Some(g) = playlist.iter_mut().find(|g| g.title.as_ref() == pg_override.as_str()) {
+            item.header.category_id = g.id;
+            item.header.group = g.title.clone();
+            g.channels.push(item);
+        } else {
+            let next_id = playlist.iter().map(|g| g.id).max().unwrap_or(0) + 1;
+            item.header.category_id = next_id;
+            item.header.group = pg_override.as_str().intern();
+            playlist.push(PlaylistGroup {
+                id: next_id,
+                title: pg_override.as_str().intern(),
+                channels: vec![item],
+                xtream_cluster: XtreamCluster::Live,
+            });
+        }
+    }
+}
+
+fn materialize_groups(
+    candidates: &[GroupCandidate],
+    config: &ChannelGroupsConfig,
+    target_name: &str,
+    mut slots: Vec<GroupSlot>,
+    playlist: &mut Vec<PlaylistGroup>,
+) -> Vec<PendingChannelGroup> {
+    let mut deferred_placements = Vec::new();
+    let mut pending_groups = Vec::with_capacity(candidates.len());
+
+    for candidate in candidates {
+        let rep_pos = candidate.positions[0];
+        let Some(rep_item) = slots.get(rep_pos.0).and_then(|g| g.channels.get(rep_pos.1)).and_then(Option::as_ref)
+        else {
+            continue;
+        };
+
+        let group_uuid = hash_string(&format!("channel-group|{target_name}|{}", candidate.key));
+        let group_header = build_group_metadata(candidate, &rep_item.header, &slots, config, group_uuid);
+
+        let mut members = Vec::with_capacity(candidate.positions.len());
+        for &pos in &candidate.positions {
+            if let Some(item) = slots.get_mut(pos.0).and_then(|g| g.channels.get_mut(pos.1)).and_then(Option::take) {
+                members.push(item);
+            }
+        }
+
+        let group_item = PlaylistItem { header: group_header };
+
+        if let Some(pg_override) = &candidate.playlist_group_override {
+            deferred_placements.push((pg_override.clone(), group_item));
+        } else if let Some(slot) = slots.get_mut(rep_pos.0).and_then(|g| g.channels.get_mut(rep_pos.1)) {
+            *slot = Some(group_item);
+        }
+
+        pending_groups.push(PendingChannelGroup {
+            group_uuid,
+            key: candidate.key.as_str().intern(),
+            origin: candidate.origin,
+            policy: candidate.policy,
+            members,
+            orphaned: candidate.orphaned.clone(),
+            epg_split: candidate.epg_split,
+        });
+    }
+
+    rebuild_playlist_from_slots(slots, deferred_placements, playlist);
+    pending_groups
+}
+
+pub(in crate::processor) fn group_playlist(
+    config: &ChannelGroupsConfig,
+    overrides: Option<&ChannelGroupOverridesDto>,
+    target_name: &str,
+    playlist: &mut Vec<PlaylistGroup>,
+) -> Vec<PendingChannelGroup> {
+    if playlist.is_empty() {
+        return Vec::new();
+    }
+
+    let candidates = collect_group_candidates(config, overrides, playlist);
+    if candidates.is_empty() {
+        return Vec::new();
+    }
+
+    let slots: Vec<GroupSlot> = playlist
+        .drain(..)
+        .map(|g| GroupSlot {
+            id: g.id,
+            title: g.title,
+            xtream_cluster: g.xtream_cluster,
+            had_channels: !g.channels.is_empty(),
+            channels: g.channels.into_iter().map(Some).collect(),
+        })
+        .collect();
+
+    materialize_groups(&candidates, config, target_name, slots, playlist)
+}

backend/processing/src/processor/channel_group/mod.rs

@@ -0,0 +1,5 @@
+mod assemble;
+#[cfg(test)]
+mod tests;
+
+pub(in crate::processor) use assemble::*;

backend/processing/src/processor/channel_group/tests.rs

@@ -0,0 +1,276 @@
+use super::*;
+use shared::{
+    model::{
+        AutoGroupOverrideDto, ChannelGroupOverridesDto, ChannelGroupsConfig, ChannelMemberRefDto, EpgConflictMode,
+        LiveStreamProperties, ManualChannelGroupDto, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType,
+        StreamProperties, XtreamCluster,
+    },
+    utils::Internable,
+};
+use tuliprox_repository::ChannelGroupOrigin;
+
+fn live(input: &str, stream_id: &str, title: &str) -> PlaylistItem {
+    let mut header = PlaylistItemHeader {
+        input_name: input.intern(),
+        id: stream_id.intern(),
+        title: title.intern(),
+        name: title.intern(),
+        url: format!("http://{input}.invalid/live/u/p/{stream_id}.ts").intern(),
+        item_type: PlaylistItemType::Live,
+        xtream_cluster: XtreamCluster::Live,
+        ..Default::default()
+    };
+    header.freeze_input_stream_id();
+    header.gen_uuid();
+    PlaylistItem { header }
+}
+
+fn group(title: &str, channels: Vec<PlaylistItem>) -> PlaylistGroup {
+    PlaylistGroup { id: 1, title: title.intern(), channels, xtream_cluster: XtreamCluster::Live }
+}
+
+fn cfg() -> ChannelGroupsConfig { ChannelGroupsConfig::default() }
+
+fn with_epg(mut item: PlaylistItem, epg: &str) -> PlaylistItem {
+    item.header.epg_channel_id = Some(epg.intern());
+    item
+}
+
+fn overrides_with_manual(id: &str, members: &[(&str, &str)]) -> ChannelGroupOverridesDto {
+    let mut overrides = ChannelGroupOverridesDto::new("T");
+    overrides.groups.push(ManualChannelGroupDto {
+        id: id.into(),
+        name: id.into(),
+        members: members
+            .iter()
+            .map(|(input, stream_id)| ChannelMemberRefDto { input: (*input).into(), stream_id: (*stream_id).into() })
+            .collect(),
+        ..Default::default()
+    });
+    overrides
+}
+
+fn set_archive(item: &mut PlaylistItem, duration: i32) {
+    let props = LiveStreamProperties { tv_archive: Some(1), tv_archive_duration: Some(duration), ..Default::default() };
+    item.header.additional_properties = Some(StreamProperties::Live(Box::new(props)));
+}
+
+fn archive_of(item: &PlaylistItem) -> Option<i32> {
+    match &item.header.additional_properties {
+        Some(StreamProperties::Live(live)) => live.tv_archive_duration,
+        _ => None,
+    }
+}
+
+#[test]
+fn auto_groups_same_caption_across_inputs() {
+    let mut playlist = vec![group(
+        "News",
+        vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD"), live("solis", "2", "ARD")],
+    )];
+    let pending = group_playlist(&cfg(), None, "T", &mut playlist);
+    assert_eq!(pending.len(), 1);
+    assert_eq!(pending[0].members.len(), 2);
+    assert_eq!(pending[0].origin, ChannelGroupOrigin::Auto);
+    let titles: Vec<_> = playlist[0].channels.iter().map(|c| c.header.title.to_string()).collect();
+    assert_eq!(titles, vec!["ZDF HD", "ARD"]); // group item at representative position
+    assert_eq!(playlist[0].channels[0].header.uuid, pending[0].group_uuid);
+}
+
+#[test]
+fn single_member_key_is_not_grouped() {
+    let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("solis", "2", "ARD")])];
+    assert!(group_playlist(&cfg(), None, "T", &mut playlist).is_empty());
+    assert_eq!(playlist[0].channels.len(), 2);
+}
+
+#[test]
+fn dsl_opt_out_and_forced_key() {
+    let mut a = live("solis", "1", "ZDF HD");
+    a.header.channel_group = Some("".intern());
+    let mut b = live("korath", "7", "Zweites");
+    b.header.channel_group = Some("zdf hd".intern());
+    let c = live("veyra", "3", "ZDF");
+    let mut playlist = vec![group("News", vec![a, b, c])];
+    let pending = group_playlist(&cfg(), None, "T", &mut playlist);
+    assert_eq!(pending.len(), 1);
+    let inputs: Vec<_> = pending[0].members.iter().map(|m| m.header.input_name.to_string()).collect();
+    assert_eq!(inputs, vec!["korath", "veyra"]);
+}
+
+#[test]
+fn manual_group_wins_over_auto_and_keeps_order() {
+    let overrides = overrides_with_manual("zdf", &[("korath", "7"), ("solis", "1")]);
+    let mut playlist = vec![group(
+        "News",
+        vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD"), live("veyra", "3", "ZDF")],
+    )];
+    let pending = group_playlist(&cfg(), Some(&overrides), "T", &mut playlist);
+    let manual = pending.iter().find(|p| p.origin == ChannelGroupOrigin::Manual).expect("manual");
+    let inputs: Vec<_> = manual.members.iter().map(|m| m.header.input_name.to_string()).collect();
+    assert_eq!(inputs, vec!["korath", "solis"]);
+    // veyra alone is a single member and stays a normal channel
+    assert!(playlist[0].channels.iter().any(|c| c.header.input_name.as_ref() == "veyra"));
+}
+
+#[test]
+fn excluded_member_never_grouped() {
+    let mut overrides = ChannelGroupOverridesDto::new("T");
+    overrides.excluded.push(ChannelMemberRefDto { input: "solis".into(), stream_id: "1".into() });
+    let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])];
+    assert!(group_playlist(&cfg(), Some(&overrides), "T", &mut playlist).is_empty());
+}
+
+#[test]
+fn auto_override_orders_and_marks_edited() {
+    let mut overrides = ChannelGroupOverridesDto::new("T");
+    overrides.auto_overrides.push(AutoGroupOverrideDto {
+        auto_key: "zdf".into(),
+        order: vec![ChannelMemberRefDto { input: "korath".into(), stream_id: "7".into() }],
+        ..Default::default()
+    });
+    let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])];
+    let pending = group_playlist(&cfg(), Some(&overrides), "T", &mut playlist);
+    assert_eq!(pending[0].origin, ChannelGroupOrigin::AutoEdited);
+    assert_eq!(pending[0].members[0].header.input_name.as_ref(), "korath");
+    assert_eq!(playlist[0].channels[0].header.title.as_ref(), "ZDF FHD"); // representative
+}
+
+#[test]
+fn orphaned_manual_member_is_skipped() {
+    let overrides = overrides_with_manual("zdf", &[("solis", "1"), ("korath", "7"), ("gone", "99")]);
+    let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])];
+    let pending = group_playlist(&cfg(), Some(&overrides), "T", &mut playlist);
+    assert_eq!(pending[0].members.len(), 2);
+    assert_eq!(pending[0].orphaned, vec![ChannelMemberRefDto { input: "gone".into(), stream_id: "99".into() }]);
+}
+
+#[test]
+fn manual_group_with_one_member_dissolves() {
+    let overrides = overrides_with_manual("zdf", &[("solis", "1"), ("gone", "99")]);
+    let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("veyra", "2", "ARD")])];
+    assert!(group_playlist(&cfg(), Some(&overrides), "T", &mut playlist).is_empty());
+    assert_eq!(playlist[0].channels.len(), 2);
+}
+
+#[test]
+fn archive_is_max_of_members() {
+    let mut a = live("solis", "1", "ZDF HD");
+    set_archive(&mut a, 3);
+    let mut b = live("korath", "7", "ZDF FHD");
+    set_archive(&mut b, 7);
+    let mut playlist = vec![group("News", vec![a, b])];
+    group_playlist(&cfg(), None, "T", &mut playlist);
+    assert_eq!(archive_of(&playlist[0].channels[0]), Some(7));
+}
+
+#[test]
+fn vod_and_series_are_never_grouped() {
+    let mut a = live("solis", "1", "Film");
+    a.header.item_type = PlaylistItemType::Video;
+    a.header.xtream_cluster = XtreamCluster::Video;
+    let mut b = live("korath", "2", "Film");
+    b.header.item_type = PlaylistItemType::Video;
+    b.header.xtream_cluster = XtreamCluster::Video;
+    let mut playlist = vec![PlaylistGroup {
+        id: 1,
+        title: "Movies".intern(),
+        channels: vec![a, b],
+        xtream_cluster: XtreamCluster::Video,
+    }];
+    assert!(group_playlist(&cfg(), None, "T", &mut playlist).is_empty());
+}
+
+#[test]
+fn different_epg_ids_split_the_auto_bucket() {
+    let mut playlist = vec![group(
+        "Sport",
+        vec![
+            with_epg(live("solis", "1", "Sport 1 HD"), "sport1.de"),
+            with_epg(live("korath", "7", "Sport 1"), "sport1.de"),
+            with_epg(live("veyra", "3", "Sport 1 FHD"), "sport1.uk"),
+            with_epg(live("orun", "4", "Sport 1"), "sport1.uk"),
+            live("solis", "9", "Sport 1"), // no EPG id → stays a normal channel
+        ],
+    )];
+    let pending = group_playlist(&cfg(), None, "T", &mut playlist);
+    assert_eq!(pending.len(), 2);
+    assert!(pending.iter().all(|p| p.epg_split && p.members.len() == 2));
+    assert!(playlist[0]
+        .channels
+        .iter()
+        .any(|c| c.header.input_name.as_ref() == "solis" && c.header.id.as_ref() == "9"));
+}
+
+#[test]
+fn epg_conflict_ignore_keeps_one_group() {
+    let mut config = cfg();
+    config.epg_conflict = EpgConflictMode::Ignore;
+    let mut playlist = vec![group(
+        "Sport",
+        vec![
+            with_epg(live("solis", "1", "Sport 1"), "sport1.de"),
+            with_epg(live("korath", "7", "Sport 1"), "sport1.uk"),
+        ],
+    )];
+    let pending = group_playlist(&config, None, "T", &mut playlist);
+    assert_eq!((pending.len(), pending[0].epg_split), (1, false));
+}
+
+#[test]
+fn dsl_forced_key_is_never_split() {
+    let mut a = with_epg(live("solis", "1", "Eins"), "a.de");
+    a.header.channel_group = Some("eins".intern());
+    let mut b = with_epg(live("korath", "7", "One"), "b.uk");
+    b.header.channel_group = Some("eins".intern());
+    let mut playlist = vec![group("News", vec![a, b])];
+    let pending = group_playlist(&cfg(), None, "T", &mut playlist);
+    assert_eq!((pending.len(), pending[0].members.len()), (1, 2));
+}
+
+#[test]
+fn match_normalize_groups_prefixed_and_unprefixed_captions() {
+    let mut config = cfg();
+    config.match_normalize.strip_regex = Some(r"^\s*(?:\|[A-Z]{2,3}\||[A-Z]{2,3}\s*[:|])\s*".to_string());
+    config.match_normalize.ignore_tokens = vec!["hevc".to_string()];
+    config.prepare().expect("valid");
+    let mut playlist = vec![group("News", vec![live("solis", "1", "DE: ZDF HEVC"), live("korath", "7", "ZDF")])];
+    let pending = group_playlist(&config, None, "T", &mut playlist);
+    assert_eq!((pending.len(), pending[0].members.len()), (1, 2));
+}
+
+#[test]
+fn name_strip_quality_cleans_derived_name_only() {
+    assert_eq!(strip_quality_words("ZDF [FHD]"), "ZDF");
+    assert_eq!(strip_quality_words("DE: ZDF HD"), "DE: ZDF");
+    assert_eq!(strip_quality_words("ZDF-HD"), "ZDF-HD");
+    assert_eq!(strip_quality_words("HD"), "HD"); // nothing left → original
+    let mut config = cfg();
+    config.name_strip_quality = true;
+    let mut playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])];
+    group_playlist(&config, None, "T", &mut playlist);
+    assert_eq!(playlist[0].channels[0].header.title.as_ref(), "ZDF");
+}
+
+#[test]
+fn emptied_playlist_group_is_dropped_and_group_uuid_is_stable() {
+    let build =
+        || vec![group("A", vec![live("solis", "1", "ZDF HD")]), group("B", vec![live("korath", "7", "ZDF FHD")])];
+    let mut first = build();
+    let p1 = group_playlist(&cfg(), None, "T", &mut first);
+    let mut second = build();
+    let p2 = group_playlist(&cfg(), None, "T", &mut second);
+    assert_eq!(first.len(), 1);
+    assert_eq!(p1[0].group_uuid, p2[0].group_uuid);
+}
+
+#[test]
+fn members_keep_original_uuid_and_group_item_is_marked() {
+    let a = live("solis", "1", "ZDF HD");
+    let b = live("korath", "7", "ZDF FHD");
+    let (uuid_a, uuid_b) = (a.header.uuid, b.header.uuid);
+    let mut playlist = vec![group("News", vec![a, b])];
+    let pending = group_playlist(&cfg(), None, "T", &mut playlist);
+    assert_eq!(pending[0].members.iter().map(|m| m.header.uuid).collect::<Vec<_>>(), vec![uuid_a, uuid_b]);
+    assert_eq!(playlist[0].channels[0].header.channel_group_uuid, Some(pending[0].group_uuid));
+}

backend/processing/src/processor/deduplicate.rs

@@ -1,45 +1,20 @@
+use super::match_key::{dedup_match_by, extract_normalized_match_key, raw_match_value};
 use shared::{
-    model::{DeduplicateConfig, DeduplicateKeep, DeduplicateMatchBy, PlaylistGroup, PlaylistItem, XtreamCluster},
-    utils::{deunicode_string, quality_rank, quality_tokens, token_quality},
+    model::{DeduplicateConfig, DeduplicateKeep, PlaylistGroup, PlaylistItem, UUIDType, XtreamCluster},
+    utils::quality_rank,
 };
-use std::collections::HashMap;
-
-/// Lowercased token join with quality tokens removed, so "News HD" and
-/// "NEWS [FHD]" produce the same key.
-fn normalized_dedup_key(value: &str, match_as_ascii: bool) -> String {
-    let value = if match_as_ascii { deunicode_string(value) } else { std::borrow::Cow::Borrowed(value) };
-    let mut key = String::with_capacity(value.len());
-    for token in quality_tokens(&value) {
-        if token_quality(token).is_some() {
-            continue;
-        }
-        if !key.is_empty() {
-            key.push(' ');
-        }
-        for ch in token.chars() {
-            key.extend(ch.to_lowercase());
-        }
-    }
-    key
-}
+use std::collections::{HashMap, HashSet};
 
 fn match_value(config: DeduplicateConfig, item: &PlaylistItem) -> String {
-    normalized_dedup_key(raw_match_value(config, item), config.match_as_ascii)
+    extract_normalized_match_key(&item.header, dedup_match_by(config.match_by), config.match_as_ascii, None)
 }
 
-fn raw_match_value(config: DeduplicateConfig, item: &PlaylistItem) -> &str {
-    let header = &item.header;
-    match config.match_by {
-        DeduplicateMatchBy::Caption => {
-            if header.title.is_empty() {
-                header.name.as_ref()
-            } else {
-                header.title.as_ref()
-            }
-        }
-        DeduplicateMatchBy::Name => header.name.as_ref(),
-        DeduplicateMatchBy::Title => header.title.as_ref(),
-    }
+fn item_raw_match_value(config: DeduplicateConfig, item: &PlaylistItem) -> &str {
+    raw_match_value(&item.header, dedup_match_by(config.match_by))
+}
+
+fn should_skip(channel: &PlaylistItem, skip: &HashSet<UUIDType>) -> bool {
+    skip.contains(&channel.header.uuid) || channel.header.channel_group_uuid.is_some()
 }
 
 /// Collapse duplicate channels across the whole target playlist (per cluster).
@@ -48,16 +23,20 @@ fn raw_match_value(config: DeduplicateConfig, item: &PlaylistItem) -> &str {
 pub(in crate::processor) fn deduplicate_playlist(
     config: DeduplicateConfig,
     playlist: &mut Vec<PlaylistGroup>,
+    skip: &HashSet<UUIDType>,
 ) -> usize {
     // winner per key: (quality rank, group index, channel index)
     let mut winners: HashMap<(XtreamCluster, String), (u8, usize, usize)> = HashMap::new();
     for (group_idx, group) in playlist.iter().enumerate() {
         for (channel_idx, channel) in group.channels.iter().enumerate() {
+            if should_skip(channel, skip) {
+                continue;
+            }
             let key_value = match_value(config, channel);
             if key_value.is_empty() {
                 continue;
             }
-            let rank = quality_rank(raw_match_value(config, channel));
+            let rank = quality_rank(item_raw_match_value(config, channel));
             match winners.entry((group.xtream_cluster, key_value)) {
                 std::collections::hash_map::Entry::Vacant(entry) => {
                     entry.insert((rank, group_idx, channel_idx));
@@ -81,6 +60,9 @@ pub(in crate::processor) fn deduplicate_playlist(
         group.channels.retain(|channel| {
             let idx = channel_idx;
             channel_idx += 1;
+            if should_skip(channel, skip) {
+                return true;
+            }
             let key_value = match_value(config, channel);
             if key_value.is_empty() {
                 return true;
@@ -104,7 +86,10 @@ pub(in crate::processor) fn deduplicate_playlist(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use shared::{model::PlaylistItemHeader, utils::Internable};
+    use shared::{
+        model::{DeduplicateMatchBy, PlaylistItemHeader},
+        utils::Internable,
+    };
 
     fn make_item(title: &str) -> PlaylistItem {
         PlaylistItem { header: PlaylistItemHeader { title: title.intern(), ..Default::default() } }
@@ -134,7 +119,7 @@ mod tests {
             keep: DeduplicateKeep::BestQuality,
             match_as_ascii: false,
         };
-        let removed = deduplicate_playlist(config, &mut playlist);
+        let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new());
         assert_eq!(removed, 2);
         let titles: Vec<_> = playlist[0].channels.iter().map(|c| c.header.title.to_string()).collect();
         assert_eq!(titles, vec!["News [FHD]", "Sports HD"]);
@@ -150,7 +135,7 @@ mod tests {
             match_as_ascii: false,
         };
 
-        let removed = deduplicate_playlist(config, &mut playlist);
+        let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new());
 
         // Target-wide curation evaluates only this surviving winner.
         assert_eq!(removed, 1);
@@ -167,7 +152,7 @@ mod tests {
             keep: DeduplicateKeep::First,
             match_as_ascii: false,
         };
-        let removed = deduplicate_playlist(config, &mut playlist);
+        let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new());
         assert_eq!(removed, 1);
         assert_eq!(playlist[0].channels[0].header.title.as_ref(), "News HD");
     }
@@ -184,7 +169,7 @@ mod tests {
             keep: DeduplicateKeep::BestQuality,
             match_as_ascii: false,
         };
-        let removed = deduplicate_playlist(config, &mut playlist);
+        let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new());
         assert_eq!(removed, 1);
         // "A" was emptied by dedup and dropped, the already-empty group survives
         assert_eq!(playlist.len(), 2);
@@ -203,7 +188,7 @@ mod tests {
             keep: DeduplicateKeep::BestQuality,
             match_as_ascii: true,
         };
-        let removed = deduplicate_playlist(config, &mut playlist);
+        let removed = deduplicate_playlist(config, &mut playlist, &HashSet::new());
         assert_eq!(removed, 1);
         assert_eq!(playlist[0].channels[0].header.title.as_ref(), "Cafe FHD");
 
@@ -214,6 +199,17 @@ mod tests {
             keep: DeduplicateKeep::BestQuality,
             match_as_ascii: false,
         };
-        assert_eq!(deduplicate_playlist(config, &mut playlist), 0);
+        assert_eq!(deduplicate_playlist(config, &mut playlist, &HashSet::new()), 0);
+    }
+
+    #[test]
+    fn dedup_never_drops_group_item() {
+        let group_item = make_item("ZDF HD");
+        let excluded = make_item("ZDF UHD");
+        let skip: HashSet<UUIDType> = [group_item.header.uuid].into_iter().collect();
+        let mut playlist = vec![make_group("News", vec![group_item, excluded])];
+        let removed = deduplicate_playlist(DeduplicateConfig::default(), &mut playlist, &skip);
+        assert_eq!(removed, 0);
+        assert_eq!(playlist[0].channels.len(), 2);
     }
 }

backend/processing/src/processor/match_key.rs

@@ -0,0 +1,195 @@
+use regex::Regex;
+use shared::{
+    model::{ChannelMatchBy, ChannelMatchNormalize, DeduplicateMatchBy, PlaylistItemHeader},
+    utils::{deunicode_string, quality_tokens, token_quality},
+};
+use std::{
+    borrow::Cow,
+    collections::{HashMap, HashSet},
+};
+
+pub(in crate::processor) const fn dedup_match_by(value: DeduplicateMatchBy) -> ChannelMatchBy {
+    match value {
+        DeduplicateMatchBy::Caption => ChannelMatchBy::Caption,
+        DeduplicateMatchBy::Name => ChannelMatchBy::Name,
+        DeduplicateMatchBy::Title => ChannelMatchBy::Title,
+    }
+}
+
+/// Lowercased token join with quality tokens removed, so "News HD" and
+/// "NEWS [FHD]" produce the same key.
+pub(in crate::processor) fn normalized_match_key(value: &str, match_as_ascii: bool) -> String {
+    let value = if match_as_ascii { deunicode_string(value) } else { std::borrow::Cow::Borrowed(value) };
+    let mut key = String::with_capacity(value.len());
+    for token in quality_tokens(&value) {
+        if token_quality(token).is_some() {
+            continue;
+        }
+        if !key.is_empty() {
+            key.push(' ');
+        }
+        for ch in token.chars() {
+            key.extend(ch.to_lowercase());
+        }
+    }
+    key
+}
+
+pub(in crate::processor) fn raw_match_value(header: &PlaylistItemHeader, match_by: ChannelMatchBy) -> &str {
+    match match_by {
+        ChannelMatchBy::Caption => {
+            if header.title.is_empty() {
+                header.name.as_ref()
+            } else {
+                header.title.as_ref()
+            }
+        }
+        ChannelMatchBy::Name => header.name.as_ref(),
+        ChannelMatchBy::Title => header.title.as_ref(),
+        ChannelMatchBy::EpgChannelId => header.epg_channel_id.as_deref().unwrap_or(""),
+    }
+}
+
+pub(in crate::processor) fn extract_normalized_match_key(
+    header: &PlaylistItemHeader,
+    match_by: ChannelMatchBy,
+    match_as_ascii: bool,
+    normalizer: Option<&MatchKeyNormalizer<'_>>,
+) -> String {
+    let raw = raw_match_value(header, match_by);
+    if match_by == ChannelMatchBy::EpgChannelId {
+        return raw.trim().to_lowercase();
+    }
+    match normalizer {
+        Some(normalizer) => normalizer.key(raw),
+        None => normalized_match_key(raw, match_as_ascii),
+    }
+}
+
+#[allow(dead_code)]
+pub(in crate::processor) struct MatchKeyNormalizer<'a> {
+    strip: Option<&'a Regex>,
+    ignore: HashSet<String>,
+    aliases: HashMap<String, String>,
+    match_as_ascii: bool,
+}
+
+#[allow(dead_code)]
+impl<'a> MatchKeyNormalizer<'a> {
+    pub(in crate::processor) fn new(config: &'a ChannelMatchNormalize, match_as_ascii: bool) -> Self {
+        let mut normalizer = Self {
+            strip: config.t_strip_regex.as_deref(),
+            ignore: config.ignore_tokens.iter().map(|token| token.to_lowercase()).collect(),
+            aliases: HashMap::new(),
+            match_as_ascii,
+        };
+        let aliases =
+            config.aliases.iter().map(|(from, to)| (normalizer.base_key(from), normalizer.base_key(to))).collect();
+        normalizer.aliases = aliases;
+        normalizer
+    }
+
+    pub(in crate::processor) fn key(&self, value: &str) -> String {
+        let key = self.base_key(value);
+        self.aliases.get(&key).cloned().unwrap_or(key)
+    }
+
+    fn base_key(&self, value: &str) -> String {
+        let stripped = self.strip.map_or(Cow::Borrowed(value), |regex| regex.replace_all(value, ""));
+        let folded = if self.match_as_ascii { deunicode_string(&stripped) } else { stripped };
+        let mut key = String::with_capacity(folded.len());
+        for token in quality_tokens(&folded) {
+            let lower = token.to_lowercase();
+            if token_quality(token).is_some() || self.ignore.contains(&lower) {
+                continue;
+            }
+            if !key.is_empty() {
+                key.push(' ');
+            }
+            key.push_str(&lower);
+        }
+        key
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use shared::{
+        model::{ChannelGroupsConfig, ChannelMatchNormalize, PlaylistItemHeader},
+        utils::Internable,
+    };
+
+    fn header(title: &str, name: &str, epg: Option<&str>) -> PlaylistItemHeader {
+        PlaylistItemHeader {
+            title: title.intern(),
+            name: name.intern(),
+            epg_channel_id: epg.map(Internable::intern),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn strips_quality_tokens_and_lowercases() {
+        assert_eq!(normalized_match_key("News HD", false), "news");
+        assert_eq!(normalized_match_key("NEWS [FHD]", false), normalized_match_key("news 1080p", false));
+    }
+
+    #[test]
+    fn caption_falls_back_to_name() {
+        assert_eq!(
+            extract_normalized_match_key(&header("", "Sport HD", None), ChannelMatchBy::Caption, false, None),
+            "sport"
+        );
+    }
+
+    #[test]
+    fn epg_channel_id_is_used_verbatim_lowercased() {
+        let h = header("ZDF HD", "ZDF", Some("ZDF.de"));
+        assert_eq!(extract_normalized_match_key(&h, ChannelMatchBy::EpgChannelId, false, None), "zdf.de");
+        assert_eq!(
+            extract_normalized_match_key(&header("X", "X", None), ChannelMatchBy::EpgChannelId, false, None),
+            ""
+        );
+    }
+
+    #[test]
+    fn ascii_folding() {
+        assert_eq!(normalized_match_key("Café HD", true), normalized_match_key("Cafe FHD", true));
+    }
+
+    fn normalize(strip: Option<&str>, ignore: &[&str], aliases: &[(&str, &str)]) -> ChannelMatchNormalize {
+        let mut cfg = ChannelGroupsConfig {
+            match_normalize: ChannelMatchNormalize {
+                strip_regex: strip.map(str::to_string),
+                ignore_tokens: ignore.iter().map(|t| (*t).to_string()).collect(),
+                aliases: aliases.iter().map(|(k, v)| ((*k).to_string(), (*v).to_string())).collect(),
+                ..Default::default()
+            },
+            ..ChannelGroupsConfig::default()
+        };
+        cfg.prepare().expect("valid");
+        cfg.match_normalize
+    }
+
+    #[test]
+    fn normalizer_strips_prefix_ignores_tokens_and_resolves_aliases() {
+        let cfg = normalize(
+            Some(r"^\s*(?:\|[A-Z]{2,3}\||[A-Z]{2,3}\s*[:|])\s*"),
+            &["HEVC", "50fps"],
+            &[("Das Erste HD", "ARD")],
+        );
+        let normalizer = MatchKeyNormalizer::new(&cfg, false);
+        assert_eq!(normalizer.key("DE: ZDF HEVC"), "zdf");
+        assert_eq!(normalizer.key("|DE| ZDF 50FPS"), "zdf");
+        assert_eq!(normalizer.key("ZDF"), "zdf");
+        assert_eq!(normalizer.key("ZDF NEO"), "zdf neo");
+        assert_eq!(normalizer.key("DE: Das Erste"), "ard");
+    }
+
+    #[test]
+    fn dedup_keys_unchanged_without_normalizer() {
+        let h = header("DE: ZDF HEVC", "", None);
+        assert_eq!(extract_normalized_match_key(&h, ChannelMatchBy::Caption, false, None), "de zdf hevc");
+    }
+}

backend/processing/src/processor/mod.rs

@@ -3,9 +3,11 @@ mod stalker;
 mod stalker_refresh;
 mod xtream;
 // mod affix;
+pub(in crate::processor) mod channel_group;
 mod deduplicate;
 mod epg;
 mod library;
+pub(in crate::processor) mod match_key;
 mod probe_handle_guard;
 mod providers;
 mod resolve_options;

backend/processing/src/processor/playlist/mod.rs

@@ -59,8 +59,8 @@ use tuliprox_curation::{
     SelectorOutcome,
 };
 use tuliprox_repository::{
-    persist_playlist_views, CategoryKey, MemoryPlaylistSource, PlaylistPublicationPlan, PlaylistSource,
-    PlaylistStorageState, TargetPlaylistPersistOptions,
+    persist_playlist_views_with_groups, CategoryKey, MemoryPlaylistSource, PendingChannelGroup,
+    PlaylistPublicationPlan, PlaylistSource, PlaylistStorageState, TargetPlaylistPersistOptions,
 };
 use tuliprox_session::ActiveProviderManager;
 

backend/processing/src/processor/playlist/target.rs

@@ -414,13 +414,14 @@ fn finalization_step_label(view: &str, stage: &str) -> String {
 pub(crate) fn prepare_eligible_catalog(
     target: &ConfigTarget,
     playlist: Vec<PlaylistGroup>,
+    group_uuids: &HashSet<UUIDType>,
     step: &mut StepMeasure,
 ) -> Vec<PlaylistGroup> {
     let mut playlist = flatten_groups(playlist);
     step.tick("playlist merge");
     log_memory_snapshot(format!("target '{}' after_playlist_merge", target.name).as_str());
     if let Some(dedup_config) = target.execution_plan.post_merge_content_dedup.as_ref() {
-        let removed = crate::processor::deduplicate::deduplicate_playlist(*dedup_config, &mut playlist);
+        let removed = crate::processor::deduplicate::deduplicate_playlist(*dedup_config, &mut playlist, group_uuids);
         if removed > 0 {
             info!("Deduplicated {removed} eligible channels for target {}", target.name);
         }
@@ -506,6 +507,29 @@ pub(crate) async fn finalize_prepared_target<E: EventSink + Clone + 'static, M:
     finalize_prepared_target_with_tmdb(ctx, prepared, tmdb_client.as_ref()).await
 }
 
+pub(super) async fn process_target_channel_groups(
+    config: &tuliprox_core::model::AppConfig,
+    target: &ConfigTarget,
+    playlist: &mut Vec<PlaylistGroup>,
+    step: &mut StepMeasure,
+    errors: &mut Vec<TuliproxError>,
+) -> Vec<PendingChannelGroup> {
+    let Some(target_cfg) = target.execution_plan.channel_groups.as_ref() else {
+        return Vec::new();
+    };
+    let cg_cfg = config.config.load().channel_groups.for_target(target_cfg);
+    let overrides =
+        tuliprox_repository::load_channel_group_overrides(config, &target.name).await.unwrap_or_else(|err| {
+            errors.push(err);
+            None
+        });
+    step.broadcast("Processing channel groups for '{}' playlist", &target.name);
+    let groups = crate::processor::channel_group::group_playlist(&cg_cfg, overrides.as_ref(), &target.name, playlist);
+    step.tick("channel grouping");
+    log_memory_snapshot(format!("target '{}' after_channel_grouping", target.name).as_str());
+    groups
+}
+
 pub(super) async fn finalize_prepared_target_with_tmdb<E: EventSink + Clone + 'static, M: MetadataUpdateSink>(
     ctx: Arc<PlaylistProcessingContext<E, M>>,
     prepared: PreparedTarget,
@@ -517,6 +541,10 @@ pub(super) async fn finalize_prepared_target_with_tmdb<E: EventSink + Clone + 's
     let mut errors = Vec::new();
     let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order);
     let mut step = StepMeasure::new(&target.name, broadcast_step);
+
+    let pending_channel_groups =
+        process_target_channel_groups(&ctx.config, target, &mut new_playlist, &mut step, &mut errors).await;
+
     if target.favourites.is_some() {
         step.broadcast("Processing favourites for '{}' playlist", &target.name);
         process_favourites(&mut new_playlist, target.favourites.as_deref());
@@ -531,8 +559,11 @@ pub(super) async fn finalize_prepared_target_with_tmdb<E: EventSink + Clone + 's
         return (Ok(()), errors);
     }
 
-    let eligible_catalog = prepare_eligible_catalog(target, new_playlist, &mut step);
-    let views = prepare_target_playlist_views(&ctx.client, tmdb_client, target, eligible_catalog).await;
+    let group_uuids: HashSet<UUIDType> = pending_channel_groups.iter().map(|p| p.group_uuid).collect();
+    let eligible_catalog = prepare_eligible_catalog(target, new_playlist, &group_uuids, &mut step);
+    let mut views = prepare_target_playlist_views(&ctx.client, tmdb_client, target, eligible_catalog).await;
+    views.channel_groups = pending_channel_groups;
+
     if views.base.is_empty()
         && views.xtream.is_none()
         && views.publication_plan == PlaylistPublicationPlan::Ordinary
@@ -577,7 +608,7 @@ pub(super) async fn finalize_prepared_target_with_tmdb<E: EventSink + Clone + 's
         flatten_tvguide(new_epg)
     };
     let library_empty = library_empty.for_filtered_playlist(&finalized_base);
-    let result = persist_playlist_views(
+    let result = persist_playlist_views_with_groups(
         &ctx.config,
         &mut finalized_base,
         finalized_xtream.as_deref_mut(),
@@ -589,6 +620,7 @@ pub(super) async fn finalize_prepared_target_with_tmdb<E: EventSink + Clone + 's
             library_empty,
             publication_plan: views.publication_plan,
         },
+        &mut views.channel_groups,
     )
     .await;
     let watch_playlist = target_watch_view(&finalized_base, finalized_xtream.as_deref());
@@ -901,6 +933,7 @@ pub(crate) struct TargetPlaylistViews {
     pub(crate) base: Vec<PlaylistGroup>,
     pub(crate) xtream: Option<Vec<PlaylistGroup>>,
     pub(crate) publication_plan: PlaylistPublicationPlan,
+    pub(crate) channel_groups: Vec<PendingChannelGroup>,
 }
 
 pub(crate) fn build_curated_playlist_views(
@@ -926,6 +959,7 @@ pub(crate) fn build_curated_playlist_views(
             has_xtream_output && !config.include_xtream_base_categories,
             appearance_filter_configured,
         ),
+        channel_groups: Vec::new(),
     }
 }
 
@@ -940,6 +974,7 @@ pub(crate) async fn prepare_target_playlist_views(
             base: playlist,
             xtream: None,
             publication_plan: PlaylistPublicationPlan::Ordinary,
+            channel_groups: Vec::new(),
         };
     };
 
@@ -954,13 +989,21 @@ pub(super) fn curation_playlist_views(
     outcome: CurationRunOutcome,
 ) -> TargetPlaylistViews {
     match outcome {
-        CurationRunOutcome::NotConfigured => {
-            TargetPlaylistViews { base: playlist, xtream: None, publication_plan: PlaylistPublicationPlan::Ordinary }
-        }
+        CurationRunOutcome::NotConfigured => TargetPlaylistViews {
+            base: playlist,
+            xtream: None,
+            publication_plan: PlaylistPublicationPlan::Ordinary,
+            channel_groups: Vec::new(),
+        },
         CurationRunOutcome::Failed(failure) => {
             warn_curation_failure(&target.name, &failure);
             // Discovery is optional; unavailable selections must not suppress the source catalog or EPG.
-            TargetPlaylistViews { base: playlist, xtream: None, publication_plan: PlaylistPublicationPlan::Ordinary }
+            TargetPlaylistViews {
+                base: playlist,
+                xtream: None,
+                publication_plan: PlaylistPublicationPlan::Ordinary,
+                channel_groups: Vec::new(),
+            }
         }
         CurationRunOutcome::Complete(evaluation) => {
             let views = build_curated_playlist_views(
@@ -1170,3 +1213,24 @@ pub(crate) async fn process_watch<E: EventSink>(
 
     true
 }
+
+#[cfg(test)]
+pub(crate) async fn run_finalize_until_persist(
+    target: &ConfigTarget,
+    mut playlist: Vec<PlaylistGroup>,
+) -> (Vec<PlaylistGroup>, Vec<PendingChannelGroup>) {
+    let mut step = StepMeasure::new("test", |_, _| {});
+    let pending = if let Some(target_cfg) = target.execution_plan.channel_groups.as_ref() {
+        let cg_cfg = shared::model::ChannelGroupsConfig::default().for_target(target_cfg);
+        crate::processor::channel_group::group_playlist(&cg_cfg, None, &target.name, &mut playlist)
+    } else {
+        Vec::new()
+    };
+    if target.favourites.is_some() {
+        process_favourites(&mut playlist, target.favourites.as_deref());
+    }
+    let group_uuids: HashSet<UUIDType> = pending.iter().map(|p| p.group_uuid).collect();
+    let eligible = prepare_eligible_catalog(target, playlist, &group_uuids, &mut step);
+    let finalized = finalize_playlist_view(target, eligible, &mut step, "base");
+    (finalized, pending)
+}

backend/processing/src/processor/playlist/tests.rs

@@ -24,6 +24,8 @@ mod curation_effect_gate;
 
 mod watch_contexts;
 
+mod channel_groups;
+
 #[cfg(test)]
 mod quality_rejection_fallback;
 

backend/processing/src/processor/playlist/tests/channel_groups.rs

@@ -0,0 +1,122 @@
+use crate::processor::playlist::target::run_finalize_until_persist;
+use shared::{
+    model::{
+        ConfigFavouritesDto, ConfigTargetDto, ConfigTargetOptions, PlaylistGroup, PlaylistItem, PlaylistItemHeader,
+        PlaylistItemType, Prepare, XtreamCluster,
+    },
+    utils::Internable,
+};
+use tuliprox_core::model::ConfigTarget;
+
+fn live(input: &str, stream_id: &str, title: &str) -> PlaylistItem {
+    let mut header = PlaylistItemHeader {
+        input_name: input.intern(),
+        id: stream_id.intern(),
+        title: title.intern(),
+        name: title.intern(),
+        url: format!("http://{input}.invalid/live/u/p/{stream_id}.ts").intern(),
+        item_type: PlaylistItemType::Live,
+        xtream_cluster: XtreamCluster::Live,
+        ..Default::default()
+    };
+    header.freeze_input_stream_id();
+    header.gen_uuid();
+    PlaylistItem { header }
+}
+
+fn group(title: &str, channels: Vec<PlaylistItem>) -> PlaylistGroup {
+    PlaylistGroup { id: 1, title: title.intern(), channels, xtream_cluster: XtreamCluster::Live }
+}
+
+fn target_with_options(options_json: &str) -> ConfigTarget {
+    let mut options: ConfigTargetOptions = serde_json::from_str(options_json).expect("valid options");
+    options.prepare().expect("prepared");
+    let dto = ConfigTargetDto { name: "T".to_string(), options: Some(options), ..Default::default() };
+    ConfigTarget::from(&dto)
+}
+
+fn target_with_options_and_favourites(options_json: &str, group_name: &str, filter_str: &str) -> ConfigTarget {
+    let mut options: ConfigTargetOptions = serde_json::from_str(options_json).expect("valid options");
+    options.prepare().expect("prepared");
+    let mut fav = ConfigFavouritesDto {
+        cluster: XtreamCluster::Live,
+        group: group_name.intern(),
+        filter: filter_str.to_string(),
+        match_as_ascii: false,
+        t_filter: None,
+    };
+    fav.prepare(None).expect("prepared filter");
+    let dto = ConfigTargetDto {
+        name: "T".to_string(),
+        options: Some(options),
+        favourites: Some(vec![fav]),
+        ..Default::default()
+    };
+    ConfigTarget::from(&dto)
+}
+
+#[tokio::test]
+async fn favourites_copy_the_group_item_instead_of_joining_the_group() {
+    let target = target_with_options_and_favourites(
+        r#"{"channel_groups":{"enabled":true}}"#,
+        "My Favourites",
+        r#"Caption ~ "ZDF.*""#,
+    );
+    let playlist = vec![group("News", vec![live("solis", "1", "ZDF HD"), live("korath", "7", "ZDF FHD")])];
+    let (finalized, pending) = run_finalize_until_persist(&target, playlist).await;
+    assert_eq!(pending.len(), 1);
+    assert_eq!(pending[0].members.len(), 2);
+    let fav = finalized.iter().find(|g| g.title.as_ref() == "My Favourites").expect("favourites group");
+    assert_eq!(fav.channels.len(), 1);
+    assert_eq!(fav.channels[0].header.channel_group_uuid, Some(pending[0].group_uuid));
+}
+
+#[tokio::test]
+async fn grouping_runs_before_deduplicate_and_dedup_ignores_group_members() {
+    let target = target_with_options(r#"{"channel_groups":{"enabled":true},"deduplicate":{}}"#);
+    let playlist = vec![group(
+        "News",
+        vec![
+            live("solis", "1", "ZDF HD"),
+            live("korath", "7", "ZDF FHD"),
+            live("veyra", "3", "ARD HD"),
+            live("veyra", "4", "ARD"),
+        ],
+    )];
+    let (finalized, pending) = run_finalize_until_persist(&target, playlist).await;
+    assert_eq!(pending.len(), 2);
+    assert_eq!(finalized[0].channels.len(), 2);
+}
+
+#[tokio::test]
+async fn grouping_uses_global_config_with_target_overrides() {
+    let dir = tempfile::tempdir().expect("tempdir");
+    let app_config = super::curation_effect_gate::app_config(dir.path());
+    let mut global: shared::model::ChannelGroupsConfig =
+        serde_json::from_str(r#"{"match_normalize":{"ignore_tokens":["backup"]},"name_strip_quality":true}"#)
+            .expect("global config");
+    global.prepare().expect("prepared");
+    app_config.config.store(std::sync::Arc::new(tuliprox_core::model::Config {
+        storage_dir: dir.path().to_string_lossy().into_owned(),
+        channel_groups: global,
+        ..tuliprox_core::model::Config::default()
+    }));
+    let target = target_with_options(r#"{"channel_groups":{"enabled":true,"name_strip_quality":false}}"#);
+    let mut playlist = vec![group("News", vec![live("a", "1", "ZDF HD"), live("b", "2", "ZDF HD backup")])];
+    let mut step = tuliprox_core::utils::StepMeasure::new("test", |_, _| {});
+    let mut errors = Vec::new();
+
+    let pending = crate::processor::playlist::target::process_target_channel_groups(
+        &app_config,
+        &target,
+        &mut playlist,
+        &mut step,
+        &mut errors,
+    )
+    .await;
+
+    assert!(errors.is_empty());
+    assert_eq!(pending.len(), 1, "the global ignore token joins both channels");
+    let names: Vec<&str> = playlist[0].channels.iter().map(|item| item.header.name.as_ref()).collect();
+    assert_eq!(names, ["ZDF HD"], "the target keeps quality words");
+}

backend/processing/src/processor/playlist/tests/curation_effect_gate.rs

@@ -15,6 +15,7 @@ use tuliprox_core::{
     model::{ApiProxyConfig, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig},
     utils::FileLockManager,
 };
+use tuliprox_repository::persist_playlist_views;
 
 pub(super) fn app_config(storage_dir: &Path) -> Arc<AppConfig> {
     Arc::new(AppConfig {

backend/processing/src/processor/playlist/tests/curation_selection.rs

@@ -69,7 +69,7 @@ fn curation_eligible_catalog_is_merged_and_deduplicated_before_matching() {
     ];
     let mut step = StepMeasure::new("test", |_, _| {});
 
-    let eligible = prepare_eligible_catalog(&target, playlist, &mut step);
+    let eligible = prepare_eligible_catalog(&target, playlist, &std::collections::HashSet::new(), &mut step);
 
     assert_eq!(eligible.len(), 1);
     assert_eq!(eligible[0].channels.len(), 1);

backend/processing/src/processor/playlist/tests/identity_serialization.rs

@@ -220,7 +220,7 @@ fn legacy_messagepack_playlist_items_default_missing_input_stream_id() {
         },
     };
 
-    let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xa0]);
+    let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xc0, 0xc0, 0xa0]);
     let decoded_header: PlaylistItemHeader =
         rmp_serde::from_slice(&header_bytes).expect("legacy header should deserialize");
     assert!(decoded_header.input_stream_id.is_empty());
@@ -259,8 +259,9 @@ fn previous_messagepack_playlist_items_default_missing_upstream_user_agent() {
         },
     };
 
-    let header: PlaylistItemHeader = rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0]))
-        .expect("previous header should deserialize");
+    let header: PlaylistItemHeader =
+        rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0, 0xc0, 0xc0]))
+            .expect("previous header should deserialize");
     let m3u: M3uPlaylistItem =
         rmp_serde::from_slice(&serialize_without_trailing_fields(&M3uPlaylistItem::from(&source), &[0xc0]))
             .expect("previous M3U item should deserialize");

backend/processing/src/processor/stream_probe.rs

@@ -289,11 +289,21 @@ async fn prepare_generic_stream_metadata<E: EventSink>(
     }
 
     let (raw_video, raw_audio, stats) = match probe_data {
-        ProbeUrlOutcome::Success(_quality, raw_video, raw_audio, stats) => (
-            raw_video.map(|value| Arc::<str>::from(value.to_string())),
-            raw_audio.map(|value| Arc::<str>::from(value.to_string())),
-            stats,
-        ),
+        ProbeUrlOutcome::Success(_quality, raw_video, raw_audio, stats) => {
+            let rv = raw_video.map(|value| Arc::<str>::from(value.to_string()));
+            let ra = raw_audio.map(|value| Arc::<str>::from(value.to_string()));
+            if item_type.is_live() {
+                let bitrate_kbps = stats.bitrate.map_or(0, |b| b / 1000);
+                events.observe_stream_probe(
+                    Arc::clone(&input.name),
+                    Arc::from(unique_id),
+                    rv.as_deref(),
+                    ra.as_deref(),
+                    bitrate_kbps,
+                );
+            }
+            (rv, ra, stats)
+        }
         // The three failure arms are the only place the *reason* survives:
         // both outcome enums above collapse to a single `ProbeFailed`, so a
         // caller cannot tell a dead provider from a cancelled shutdown. The