diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c21271ee..0d6e312eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ## ⚠️ Breaking Changes +- **Smart EPG normalization now preserves XMLTV ID separators by default.** The default `normalize_regex` changed from + `[^a-zA-Z0-9\-]` to `[^a-zA-Z0-9._\-]`. Existing configurations that explicitly set the former pattern keep the + legacy separator-removal behavior; remove the override or use the new pattern to adopt the new default. + - **STRM names now use the processed target title by default**: Previously, STRM folders and filenames preferred the media metadata name. Existing STRM targets that must retain that behavior need `use_metadata: true`; otherwise the next export can generate different paths, and `cleanup: true` can remove the old files. diff --git a/backend/src/processing/parser/xmltv.rs b/backend/src/processing/parser/xmltv.rs index c44834d85..3b136e133 100644 --- a/backend/src/processing/parser/xmltv.rs +++ b/backend/src/processing/parser/xmltv.rs @@ -1,32 +1,27 @@ use crate::{ model::{ + EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_ATTRIB_LANG, EPG_TAG_CATEGORY, EPG_TAG_CHANNEL, EPG_TAG_DESC, + EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_LIVE, EPG_TAG_NEW, EPG_TAG_PROGRAMME, EPG_TAG_TITLE, EPG_TAG_TV, Epg, EpgSmartMatchConfig, IcsDummyPolicy, IcsEpgSourceConfig, PersistedEpgSource, PersistedEpgSourceKind, - TVGuide, XmlTag, XmlTagIcon, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_ATTRIB_LANG, EPG_TAG_CATEGORY, - EPG_TAG_CHANNEL, EPG_TAG_DESC, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_LIVE, EPG_TAG_NEW, - EPG_TAG_PROGRAMME, EPG_TAG_TITLE, EPG_TAG_TV, - }, - processing::{ - parser::ics, - processor::EpgIdCache, + TVGuide, XmlTag, XmlTagIcon, }, + processing::{parser::ics, processor::EpgIdCache}, repository::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate, FlushPolicy}, utils::{ - arc_str_serde, async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, - parse_xmltv_time, with_folded_epg_id, + arc_str_serde, async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, parse_xmltv_time, + with_folded_epg_id, }, }; use log::error; use quick_xml::events::{BytesStart, BytesText, Event}; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use shared::{ concat_string, model::{EpgCategory, EpgChannel, EpgNamePrefix, EpgProgramme}, - utils::{deunicode_string, Internable, CONSTANTS}, + utils::{CONSTANTS, Internable, deunicode_string}, }; use std::{ borrow::Cow, - cmp::min, collections::{HashMap, HashSet}, io, path::PathBuf, @@ -93,22 +88,47 @@ fn combine(join: &str, left: &str, right: &str) -> String { combined } +fn strip_markers(input: &str, markers: &[String]) -> String { + let mut output = String::with_capacity(input.len()); + let mut cursor = 0; + while cursor < input.len() { + let left_is_boundary = cursor == 0 || !input.as_bytes()[cursor - 1].is_ascii_alphanumeric(); + let matched_end = left_is_boundary + .then(|| { + markers.iter().filter(|marker| !marker.is_empty()).find_map(|marker| { + let end = cursor + marker.len(); + let candidate = input.get(cursor..end)?; + (candidate.eq_ignore_ascii_case(marker) + && input.as_bytes().get(end).is_none_or(|byte| !byte.is_ascii_alphanumeric())) + .then_some(end) + }) + }) + .flatten(); + if let Some(end) = matched_end { + cursor = end; + } else { + let next = input[cursor..].chars().next().expect("cursor is within the string"); + output.push(next); + cursor += next.len_utf8(); + } + } + output +} + /// # Panics pub fn normalize_channel_name(name: &str, normalize_config: &EpgSmartMatchConfig) -> String { let normalized = deunicode_string(name.trim()).to_lowercase(); let (channel_name, suffix) = name_prefix(&normalized, normalize_config); - // Remove all non-alphanumeric characters (except dashes and underscores). - let cleaned_name = normalize_config.normalize_regex.replace_all(channel_name, ""); - // Remove terms like resolution - let cleaned_name = normalize_config.strip.iter().fold(cleaned_name.to_string(), |acc, term| acc.replace(term, "")); - match suffix { - None => cleaned_name, + let stripped_name = strip_markers(channel_name, &normalize_config.strip); + let reconstructed = match suffix { + None => stripped_name, Some(sfx) => match &normalize_config.name_prefix { - EpgNamePrefix::Ignore => cleaned_name, - EpgNamePrefix::Suffix(sep) => combine(sep, &cleaned_name, sfx), - EpgNamePrefix::Prefix(sep) => combine(sep, sfx, &cleaned_name), + EpgNamePrefix::Ignore => stripped_name, + EpgNamePrefix::Suffix(separator) => combine(separator, &stripped_name, sfx), + EpgNamePrefix::Prefix(separator) => combine(separator, sfx, &stripped_name), }, - } + }; + normalize_config.normalize_regex.replace_all(&reconstructed, "").into_owned() } impl TVGuide { @@ -150,26 +170,6 @@ impl TVGuide { } } - fn try_fuzzy_matching(id_cache: &mut EpgIdCache, epg_id: &Arc, tag: &XmlTag, fuzzy_matching: bool) -> bool { - let mut matched = - tag.normalized_epg_ids.as_ref().is_some_and(|ids| id_cache.match_with_normalized(epg_id, ids)); - if !matched && fuzzy_matching { - let (fuzzy_matched, matched_normalized_name) = Self::find_best_fuzzy_match(id_cache, tag); - if fuzzy_matched { - if let Some(key) = matched_normalized_name { - id_cache.normalized.entry(key).and_modify(|entry| { - entry.replace(epg_id.clone()); - // Inline fold (mirrors `EpgIdCache::insert_channel_epg_id`); a - // `&mut self` call would break the disjoint field capture here. - with_folded_epg_id(epg_id, |folded| id_cache.channel_epg_id.insert(folded.intern())); - matched = true; - }); - } - } - } - matched - } - fn channel_display_name(tag: &XmlTag) -> Option> { tag.children.as_ref().and_then(|children| { children @@ -243,78 +243,6 @@ impl TVGuide { Some(programme) } - /// Finds the best fuzzy match for a channel's normalized EPG ID using phonetic encoding and Jaro-Winkler similarity. - /// - /// Iterates over the tag's normalized EPG IDs, computes their phonetic codes, and searches for candidates in the phonetics map. - /// For each candidate, calculates the Jaro-Winkler similarity score and tracks the best match above the configured threshold. - /// Returns a tuple indicating whether a suitable match was found and the matched normalized EPG ID if available. - /// - /// # Returns - /// - /// A tuple where the first element is `true` if a match above the threshold was found, and the second element is the matched normalized EPG ID. - /// - /// # Examples - /// - /// ``` - /// let (found, matched) = find_best_fuzzy_match(&mut id_cache, &tag); - /// if found { - /// println!("Best match: {:?}", matched); - /// } - /// ``` - fn find_best_fuzzy_match(id_cache: &mut EpgIdCache, tag: &XmlTag) -> (bool, Option>) { - let match_threshold = id_cache.smart_match_config.match_threshold; - let best_match_threshold = id_cache.smart_match_config.best_match_threshold; - - let Some(normalized_epg_ids) = tag.normalized_epg_ids.as_ref() else { - return (false, None); - }; - - // 1) Precalculation: (tag_normalized, tag_code) - let pre: Vec<(Arc, Arc)> = - normalized_epg_ids.iter().map(|tn| (tn.clone(), id_cache.phonetic(tn))).collect(); - - // 2) Early exit if match >= best_match_threshold - for (tag_normalized, tag_code) in &pre { - if let Some(candidates) = id_cache.phonetics.get(tag_code) { - if let Some(good_enough) = candidates.par_iter().find_any(|norm_key| { - let jw = strsim::jaro_winkler(norm_key, tag_normalized); - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let score = min(100, (jw * 100.0).round() as u16); - score >= best_match_threshold - }) { - return (true, Some(good_enough.clone())); - } - } - } - - // 3) No full match: find best match with match_threshold - let best = pre - .par_iter() - .filter_map(|(tag_normalized, tag_code)| { - id_cache.phonetics.get(tag_code).map(|candidates| { - candidates - .par_iter() - .map(|norm_key| { - let jw = strsim::jaro_winkler(norm_key, tag_normalized); - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let score = min(100, (jw * 100.0).round() as u16); - (score, norm_key) - }) - .reduce_with(|a, b| if a.0 >= b.0 { a } else { b }) - }) - }) - .flatten() - .reduce_with(|a, b| if a.0 >= b.0 { a } else { b }); - - if let Some((score, best_key)) = best { - if score >= match_threshold { - return (true, Some(Arc::clone(best_key))); - } - } - - (false, None) - } - /// Parses and filters a compressed EPG XML file, extracting relevant channel and program tags based on smart and fuzzy matching criteria. /// /// Returns an `Epg` containing filtered tags and TV attributes if any matching channels are found; otherwise, returns `None`. @@ -346,7 +274,6 @@ impl TVGuide { let mut source_processed: HashSet> = HashSet::with_capacity(5000); let mut accepted_channels = 0usize; let smart_match = id_cache.smart_match_config.enabled; - let fuzzy_matching = smart_match && id_cache.smart_match_config.fuzzy_matching; let mut filter_tags = |mut tag: XmlTag| { match tag.name.as_ref() { EPG_TAG_CHANNEL => { @@ -357,11 +284,29 @@ impl TVGuide { } Self::prepare_tag(id_cache, &mut tag, smart_match); + if smart_match && id_cache.needs_guide_names(&tag_epg_id) { + id_cache.register_guide_names( + &tag_epg_id, + tag.children.iter().flatten().filter_map(|child| { + (child.name.as_ref() == EPG_TAG_DISPLAY_NAME) + .then_some(child.value.as_ref()) + .flatten() + }), + ); + } // Case-insensitive (ASCII) membership: fold the guide id for the // lookup only; `tag_epg_id` keeps its original case for output. let add_channel = if smart_match { - id_cache.contains_channel_epg_id(&tag_epg_id) - || Self::try_fuzzy_matching(id_cache, &tag_epg_id, &tag, fuzzy_matching) + let direct_match = id_cache.contains_channel_epg_id(&tag_epg_id); + let normalized_match = tag.normalized_epg_ids.as_ref().is_some_and(|candidates| { + id_cache.match_epg_channel_candidates( + &tag_epg_id, + candidates, + epg_source.priority, + source_order, + ) + }); + direct_match || normalized_match } else { id_cache.contains_channel_epg_id(&tag_epg_id) }; @@ -432,8 +377,14 @@ impl TVGuide { let normalized_candidates = id_cache.normalize_candidates(candidates); let add_channel = if id_cache.smart_match_enabled { - id_cache.contains_channel_epg_id(channel_id) - || id_cache.match_epg_channel_candidates(channel_id, &normalized_candidates) + let direct_match = id_cache.contains_channel_epg_id(channel_id); + let normalized_match = id_cache.match_epg_channel_candidates( + channel_id, + &normalized_candidates, + epg_source.priority, + source_order, + ); + direct_match || normalized_match } else { id_cache.contains_channel_epg_id(channel_id) }; @@ -515,6 +466,12 @@ impl TVGuide { } } } + let available_epg_ids = accumulator.channel_ids_with_programmes(); + id_cache.finalize_matches(&available_epg_ids); + let mut selected_epg_ids = id_cache.selected_epg_ids(&available_epg_ids); + selected_epg_ids.extend(id_cache.channel_epg_id.iter().cloned()); + let retained_epg_ids = accumulator.retain_channels(&selected_epg_ids); + id_cache.replace_processed_epg_ids(retained_epg_ids.intersection(&available_epg_ids).cloned().collect()); accumulator.finish_epg_with_icon_overrides() } } @@ -730,6 +687,20 @@ pub(crate) struct EpgMergeAccumulator { impl EpgMergeAccumulator { pub(crate) fn new() -> Self { Self::default() } + fn channel_ids_with_programmes(&self) -> HashSet> { + self.channels + .iter() + .filter(|(id, channel)| !channel.programmes.is_empty() || self.dummy_policies.contains_key(*id)) + .map(|(id, _)| Arc::clone(id)) + .collect() + } + + fn retain_channels(&mut self, selected_epg_ids: &HashSet>) -> HashSet> { + self.channels.retain(|id, _| selected_epg_ids.contains(id)); + self.dummy_policies.retain(|id, _| selected_epg_ids.contains(id)); + self.channels.keys().cloned().collect() + } + pub(crate) fn set_attributes_if_preferred( &mut self, priority: i16, @@ -971,12 +942,7 @@ impl EpgMergeAccumulator { path.display() ); - let source = DiskEpgSource::new( - path, - attributes.map(|a| a.attributes), - source_priority, - source_order, - ); + let source = DiskEpgSource::new(path, attributes.map(|a| a.attributes), source_priority, source_order); // DiskEpgSource now owns the path; disarm the guard so its Drop does // not double-remove the file. `mem::forget` would also work, but // `take()` keeps the guard in scope and is auditable. @@ -1240,7 +1206,11 @@ pub(crate) fn merge_epg_trees(sources: Vec) -> io::Result::try_new(&source.path) .map_err(|e| io::Error::other(format!("open temp EPG tree at {}: {e}", source.path.display())))?; @@ -1267,8 +1237,8 @@ mod tests { TVGuide, }, processing::parser::xmltv::{ - flatten_tvguide, merge_epg_channels_by_priority, merge_epg_channels_by_priority_with_dummy_policies, - normalize_channel_name, EpgDummyPolicySource, EpgMergeAccumulator, + EpgDummyPolicySource, EpgMergeAccumulator, flatten_tvguide, merge_epg_channels_by_priority, + merge_epg_channels_by_priority_with_dummy_policies, normalize_channel_name, }, utils::FileLockManager, }; @@ -1325,6 +1295,35 @@ mod tests { assert_eq!(normalized, "lovenature".to_string()); } + #[test] + fn normalization_keeps_country_suffixes_consistent() { + let mut dto = EpgSmartMatchConfigDto { + enabled: true, + name_prefix: EpgNamePrefix::Suffix(".".to_string()), + ..Default::default() + }; + dto.prepare().expect("valid smart-match config"); + let config = EpgSmartMatchConfig::from(dto); + + assert_eq!(normalize_channel_name("FR: TF1 FHD", &config), "tf1.fr"); + assert_eq!(normalize_channel_name("TF1.fr", &config), "tf1.fr"); + } + + #[test] + fn normalization_strips_quality_markers_without_corrupting_names() { + let mut dto = EpgSmartMatchConfigDto { enabled: true, ..Default::default() }; + dto.prepare().expect("valid smart-match config"); + let config = EpgSmartMatchConfig::from(dto); + + assert_eq!(normalize_channel_name("RMC Story H265 50FPS", &config), "rmcstory"); + assert_eq!(normalize_channel_name("Ashdod TV HD", &config), "ashdodtv"); + } + + #[test] + fn marker_stripping_skips_non_utf8_boundaries() { + assert_eq!(super::strip_markers("éclair", &["x".to_string()]), "éclair"); + } + fn epg_channel(id: &str, title: Option<&str>, icon: Option<&str>, programmes: Vec) -> EpgChannel { EpgChannel { id: id.intern(), @@ -1451,19 +1450,13 @@ mod tests { programme }; let merged = merge_epg_channels_by_priority(vec![ - ( - 0, - vec![epg_channel("demo.channel", None, None, vec![high_programme_with_categories])], - ), + (0, vec![epg_channel("demo.channel", None, None, vec![high_programme_with_categories])]), (10, vec![epg_channel("demo.channel", None, None, vec![low_programme_extra])]), ]); assert_eq!(merged.len(), 1); let programme = &merged[0].programmes[0]; - assert_eq!( - programme.categories, - vec![EpgCategory { value: "Drama".intern(), lang: None }], - ); + assert_eq!(programme.categories, vec![EpgCategory { value: "Drama".intern(), lang: None }],); assert!(programme.is_live); assert!(programme.is_new); } @@ -1661,8 +1654,8 @@ mod tests { .expect("write XMLTV fixture"); let file_locks = Arc::new(FileLockManager::new()); - let guide = TVGuide::new(vec![xmltv_source(epg_path.clone(), 0, false)]) - .with_file_locks(Arc::clone(&file_locks)); + let guide = + TVGuide::new(vec![xmltv_source(epg_path.clone(), 0, false)]).with_file_locks(Arc::clone(&file_locks)); let mut id_cache = EpgIdCache::new(None); id_cache.insert_channel_epg_id("demo.channel"); let write_guard = file_locks.write_lock(&epg_path).await; @@ -1921,10 +1914,9 @@ mod tests { let source_order_winner = merge(vec![dummy_policy_source(0, 2, "Later source"), dummy_policy_source(0, 1, "Earlier source")]); assert!(!source_order_winner.programmes.is_empty()); - assert!(source_order_winner - .programmes - .iter() - .all(|programme| programme.title.as_deref() == Some("Earlier source"))); + assert!( + source_order_winner.programmes.iter().all(|programme| programme.title.as_deref() == Some("Earlier source")) + ); } #[ignore = "requires a local XMLTV fixture under /tmp"] @@ -2166,7 +2158,7 @@ mod tests { /// optimisation is built on sand. #[test] fn disk_path_matches_in_memory_finish() { - use super::{merge_epg_trees, EpgMergeAccumulator}; + use super::{EpgMergeAccumulator, merge_epg_trees}; // Build one source by appending one programme per channel through the // accumulator's primary entry point. `add_channel_with_programmes` is @@ -2180,7 +2172,11 @@ mod tests { for i in channels { let id: Arc = format!("ch-{i:04}").into(); let priority = if source == 0 { 5 } else { 3 }; // source 1 wins - let (start, stop) = if source == 0 { (i64::from(i), i64::from(i + 1)) } else { (100 + i64::from(i), 101 + i64::from(i)) }; + let (start, stop) = if source == 0 { + (i64::from(i), i64::from(i + 1)) + } else { + (100 + i64::from(i), 101 + i64::from(i)) + }; acc.add_channel_with_programmes( i16::try_from(priority).unwrap_or(0), source, diff --git a/backend/src/processing/processor/epg.rs b/backend/src/processing/processor/epg.rs index 492a92aa9..632eada04 100644 --- a/backend/src/processing/processor/epg.rs +++ b/backend/src/processing/processor/epg.rs @@ -1,14 +1,63 @@ -use crate::model::Epg; -use crate::model::{EpgConfig, EpgSmartMatchConfig}; -use crate::model::FetchedPlaylist; -use crate::processing::parser::xmltv::normalize_channel_name; -use crate::utils::with_folded_epg_id; +use crate::{ + model::{Epg, EpgConfig, EpgSmartMatchConfig, FetchedPlaylist}, + processing::parser::xmltv::normalize_channel_name, + utils::with_folded_epg_id, +}; use log::{debug, trace, warn}; use rphonetic::{DoubleMetaphone, Encoder}; -use std::collections::{HashMap, HashSet}; -use shared::model::{EpgSmartMatchConfigDto, PlaylistItem, XtreamCluster}; -use std::sync::Arc; -use shared::utils::Internable; +use shared::{ + model::{EpgNamePrefix, EpgSmartMatchConfigDto, PlaylistItem, XtreamCluster}, + utils::{CONSTANTS, Internable}, +}; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + sync::Arc, +}; + +const MIN_FUZZY_SCORE_MARGIN: u16 = 3; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum SmartMatchKind { + Fuzzy, + Exact, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct SmartMatchRank { + kind: SmartMatchKind, + score: u16, + source_priority: i16, + source_order: usize, +} + +#[derive(Clone, Debug)] +struct RankedEpgMatch { + epg_id: Arc, + rank: SmartMatchRank, +} + +impl SmartMatchRank { + fn cmp_preference(&self, other: &Self) -> Ordering { + self.kind + .cmp(&other.kind) + .then_with(|| self.score.cmp(&other.score)) + .then_with(|| other.source_priority.cmp(&self.source_priority)) + .then_with(|| other.source_order.cmp(&self.source_order)) + } +} + +#[derive(Debug)] +struct FuzzyCandidate { + key: Arc, + score: u16, +} + +#[derive(Debug, Default)] +struct MatchRequirement { + missing_id: bool, + direct_ids: HashSet>, +} pub struct EpgIdCache { /// Membership set of the epg ids collected from playlist channels. @@ -20,6 +69,12 @@ pub struct EpgIdCache { pub channel_epg_id: HashSet>, pub normalized: HashMap, Option>>, pub phonetics: HashMap, HashSet>>, + prefixes: HashMap>>, + cores: HashMap, HashSet>>, + match_ranks: HashMap, SmartMatchRank>, + match_candidates: HashMap, HashMap, RankedEpgMatch>>, + match_requirements: HashMap, MatchRequirement>, + guide_names: HashMap, HashSet>>, pub processed: HashSet>, pub smart_match_config: EpgSmartMatchConfig, pub metaphone: DoubleMetaphone, @@ -47,18 +102,21 @@ impl EpgIdCache { channel_epg_id: HashSet::new(), // contains the epg_ids collected from playlist channels normalized: HashMap::new(), phonetics: HashMap::new(), + prefixes: HashMap::new(), + cores: HashMap::new(), + match_ranks: HashMap::new(), + match_candidates: HashMap::new(), + match_requirements: HashMap::new(), + guide_names: HashMap::new(), processed: HashSet::new(), metaphone: DoubleMetaphone::default(), smart_match_enabled: normalize_config.enabled, fuzzy_match_enabled: normalize_config.enabled && normalize_config.fuzzy_matching, smart_match_config: normalize_config, - } } - fn is_empty(&self) -> bool { - self.channel_epg_id.is_empty() && self.normalized.is_empty() - } + fn is_empty(&self) -> bool { self.channel_epg_id.is_empty() && self.normalized.is_empty() } /// Adds an epg id to the case-folded membership set. /// @@ -75,6 +133,8 @@ impl EpgIdCache { with_folded_epg_id(id, |folded| self.channel_epg_id.contains(folded)) } + pub(crate) fn needs_guide_names(&self, id: &str) -> bool { self.contains_channel_epg_id(id) } + pub fn insert_processed_epg_id(&mut self, id: &str) { with_folded_epg_id(id, |folded| self.processed.insert(folded.intern())); } @@ -83,6 +143,75 @@ impl EpgIdCache { with_folded_epg_id(id, |folded| self.processed.contains(folded)) } + pub fn selected_epg_ids(&self, available_epg_ids: &HashSet>) -> HashSet> { + let mut selected = self.channel_epg_id.intersection(available_epg_ids).cloned().collect::>(); + selected.extend(self.normalized.iter().filter_map(|(key, id)| { + let requirement = self.match_requirements.get(key)?; + let needs_smart_match = requirement.missing_id + || requirement.direct_ids.iter().any(|direct_id| { + !available_epg_ids.contains(direct_id) + || self.normalized_id_name_compatible(direct_id, key).is_some_and(|compatible| !compatible) + }); + needs_smart_match.then(|| id.as_ref().map(|id| with_folded_epg_id(id, |folded| folded.intern()))).flatten() + })); + selected + } + + pub fn replace_processed_epg_ids(&mut self, ids: HashSet>) { self.processed = ids; } + + pub fn register_guide_names(&mut self, epg_id: &str, names: I) + where + I: IntoIterator, + S: AsRef, + { + let normalized_names = names + .into_iter() + .map(|name| self.normalize(name.as_ref()).intern()) + .filter(|name| !name.is_empty()) + .collect::>(); + if normalized_names.is_empty() { + return; + } + let folded_id = with_folded_epg_id(epg_id, |folded| folded.intern()); + self.guide_names.entry(folded_id).or_default().extend(normalized_names); + } + + pub fn finalize_matches(&mut self, available_epg_ids: &HashSet>) { + self.match_ranks.clear(); + let mut exact = 0usize; + let mut fuzzy = 0usize; + let mut ambiguous = 0usize; + let mut unavailable = 0usize; + for (key, entry) in &mut self.normalized { + let candidates = self.match_candidates.get(key); + let available_count = candidates.map_or(0, |candidates| { + candidates.keys().filter(|folded_id| available_epg_ids.contains(*folded_id)).count() + }); + let winner = + candidates.and_then(|candidates| { + best_ranked_match(candidates.iter().filter_map(|(folded_id, candidate)| { + available_epg_ids.contains(folded_id).then_some(candidate) + })) + }); + if let Some(winner) = winner { + entry.replace(Arc::clone(&winner.epg_id)); + self.match_ranks.insert(Arc::clone(key), winner.rank); + match winner.rank.kind { + SmartMatchKind::Exact => exact += 1, + SmartMatchKind::Fuzzy => fuzzy += 1, + } + } else { + entry.take(); + if available_count > 0 { + ambiguous += 1; + } else { + unavailable += 1; + } + } + } + debug!("Smart EPG candidates: exact={exact}, fuzzy={fuzzy}, ambiguous={ambiguous}, unavailable={unavailable}"); + } + /// Normalizes a channel name, computes its phonetic encoding, and stores both in the cache for later EPG matching. /// /// The normalized name is mapped to the provided EPG ID (if any), and the phonetic encoding is added to the phonetics map. @@ -96,21 +225,32 @@ impl EpgIdCache { /// assert!(cache.normalized.contains_key(&cache.normalize("Discovery Channel"))); /// ``` fn normalize_and_store(&mut self, name: &str, epg_id: Option<&Arc>) { - self.insert_normalized(name); + self.insert_normalized(name, epg_id); if let Some(chan_epg_id) = epg_id { - self.insert_normalized(chan_epg_id); + self.insert_normalized(chan_epg_id, epg_id); } } - fn insert_normalized(&mut self, key: &str) { + fn insert_normalized(&mut self, key: &str, direct_epg_id: Option<&Arc>) { let normalized = self.normalize(key).intern(); + if normalized.is_empty() { + return; + } let phonetic = self.phonetic(&normalized); - self.normalized.insert(normalized.clone(), None); - self.phonetics - .entry(phonetic.clone()) - .or_default() - .insert(normalized); + self.normalized.entry(normalized.clone()).or_insert(None); + let requirement = self.match_requirements.entry(normalized.clone()).or_default(); + if let Some(id) = direct_epg_id.filter(|id| !id.is_empty()) { + requirement.direct_ids.insert(with_folded_epg_id(id, |folded| folded.intern())); + } else { + requirement.missing_id = true; + } + self.phonetics.entry(phonetic).or_default().insert(normalized.clone()); + if let Some(prefix) = candidate_prefix(&normalized) { + self.prefixes.entry(prefix).or_default().insert(normalized.clone()); + } + let (core, _) = self.split_country(&normalized); + self.cores.entry(core.intern()).or_default().insert(normalized); } /// Returns the normalized form of a channel name using the configured smart match settings. @@ -122,17 +262,11 @@ impl EpgIdCache { /// let normalized = cache.normalize("HBO HD"); /// assert!(!normalized.is_empty()); /// ``` - fn normalize(&self, name: &str) -> String { - normalize_channel_name(name, &self.smart_match_config) - } + fn normalize(&self, name: &str) -> String { normalize_channel_name(name, &self.smart_match_config) } pub(crate) fn phonetic(&self, name: &Arc) -> Arc { let result = self.metaphone.encode(name); - if result.is_empty() { - name.clone() - } else { - result.intern() - } + if result.is_empty() { name.clone() } else { result.intern() } } pub fn collect_epg_id(&mut self, fp: &mut FetchedPlaylist) { @@ -153,7 +287,8 @@ impl EpgIdCache { // for fuzzy_matching we need to put the normalized name even if there is an epg_id, because the epg_id // could not match to the epg file. And then we try to guess it based on normalized name - let needs_normalization = smart_match_enabled && (fuzzy_matching || missing_epg_id); + let needs_normalization = + smart_match_enabled && (fuzzy_matching || missing_epg_id) && !is_decorative_channel_name(name); if needs_normalization { self.normalize_and_store(name, epg_channel_id); @@ -167,19 +302,6 @@ impl EpgIdCache { } } - pub fn match_with_normalized(&mut self, epg_id: &Arc, normalized_epg_ids: &[Arc]) -> bool { - for key in normalized_epg_ids { - if let Some(entry) = self.normalized.get_mut(key) { - entry.replace(Arc::clone(epg_id)); - // Inline fold (mirrors `insert_channel_epg_id`); a `&mut self` call - // here would conflict with the `entry` borrow of `self.normalized`. - with_folded_epg_id(epg_id, |folded| self.channel_epg_id.insert(folded.intern())); - return true; - } - } - false - } - pub fn normalize_candidates(&self, candidates: I) -> Vec> where I: IntoIterator, @@ -190,60 +312,425 @@ impl EpgIdCache { .map(|candidate| candidate.as_ref().trim().to_string()) .filter(|candidate| !candidate.is_empty()) .map(|candidate| self.normalize(&candidate).intern()) + .filter(|candidate| !candidate.is_empty()) .collect() } - pub fn match_epg_channel_candidates(&mut self, epg_id: &Arc, normalized_candidates: &[Arc]) -> bool { - let mut matched = self.match_with_normalized(epg_id, normalized_candidates); - if !matched && self.fuzzy_match_enabled { - if let Some(key) = self.find_best_fuzzy_match_for_candidates(normalized_candidates) { - self.normalized.entry(key).and_modify(|entry| { - entry.replace(epg_id.clone()); - with_folded_epg_id(epg_id, |folded| self.channel_epg_id.insert(folded.intern())); - matched = true; - }); + pub fn match_epg_channel_candidates( + &mut self, + epg_id: &Arc, + normalized_candidates: &[Arc], + source_priority: i16, + source_order: usize, + ) -> bool { + let normalized_epg_id = self.normalize(epg_id); + let (_, id_country) = self.split_country(&normalized_epg_id); + let guide_country = id_country.or_else(|| self.common_country(normalized_candidates)); + let mut exact_candidate_found = false; + let mut selected = false; + + for key in normalized_candidates { + if self.normalized.contains_key(key) { + let (_, playlist_country) = self.split_country(key); + if !countries_compatible(playlist_country, guide_country) { + continue; + } + exact_candidate_found = true; + selected |= self.assign_match( + key, + epg_id, + SmartMatchRank { kind: SmartMatchKind::Exact, score: 100, source_priority, source_order }, + ); } } - matched + + let mut relaxed_keys = HashSet::new(); + for guide_key in normalized_candidates { + let (guide_core, candidate_country) = self.split_country(guide_key); + let effective_guide_country = guide_country.or(candidate_country); + if let Some(keys) = self.cores.get(guide_core) { + relaxed_keys.extend( + keys.iter() + .filter(|key| { + let (_, playlist_country) = self.split_country(key); + countries_compatible(playlist_country, effective_guide_country) + }) + .cloned(), + ); + } + } + for key in relaxed_keys { + exact_candidate_found = true; + selected |= self.assign_match( + &key, + epg_id, + SmartMatchRank { kind: SmartMatchKind::Exact, score: 99, source_priority, source_order }, + ); + } + + if exact_candidate_found || !self.fuzzy_match_enabled { + return selected; + } + + let Some(candidate) = self.find_best_fuzzy_match(normalized_candidates, guide_country) else { + return false; + }; + let rank = + SmartMatchRank { kind: SmartMatchKind::Fuzzy, score: candidate.score, source_priority, source_order }; + self.assign_match(&candidate.key, epg_id, rank) } - fn find_best_fuzzy_match_for_candidates(&self, normalized_candidates: &[Arc]) -> Option> { - let match_threshold = self.smart_match_config.match_threshold; - let best_match_threshold = self.smart_match_config.best_match_threshold; - let pre = normalized_candidates - .iter() - .map(|candidate| (candidate.clone(), self.phonetic(candidate))) - .collect::>(); + fn assign_match(&mut self, key: &Arc, epg_id: &Arc, rank: SmartMatchRank) -> bool { + if !self.normalized.contains_key(key) { + return false; + } + let folded_id = with_folded_epg_id(epg_id, |folded| folded.intern()); + let candidates = self.match_candidates.entry(Arc::clone(key)).or_default(); + match candidates.entry(folded_id) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + if rank.cmp_preference(&entry.get().rank).is_gt() { + entry.insert(RankedEpgMatch { epg_id: Arc::clone(epg_id), rank }); + } + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(RankedEpgMatch { epg_id: Arc::clone(epg_id), rank }); + } + } - for (candidate, code) in &pre { - if let Some(keys) = self.phonetics.get(code) { - if let Some(good_enough) = keys.iter().find(|norm_key| { - let jw = strsim::jaro_winkler(norm_key, candidate); - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let score = ((jw * 100.0).round() as u16).min(100); - score >= best_match_threshold - }) { - return Some(good_enough.clone()); + let winner = candidates.values().max_by(|left, right| compare_ranked_matches(left, right)); + if let Some(winner) = winner { + self.normalized.get_mut(key).expect("normalized key exists").replace(Arc::clone(&winner.epg_id)); + self.match_ranks.insert(Arc::clone(key), winner.rank); + } + true + } + + fn find_best_fuzzy_match( + &self, + normalized_candidates: &[Arc], + guide_country: Option<&str>, + ) -> Option { + let mut phonetic_keys = HashSet::new(); + for candidate in normalized_candidates { + if let Some(keys) = self.phonetics.get(&self.phonetic(candidate)) { + phonetic_keys.extend(keys.iter().cloned()); + } + } + if let Some(best) = self.select_unambiguous_candidate( + normalized_candidates, + phonetic_keys.iter(), + self.smart_match_config.match_threshold, + guide_country, + ) { + return Some(best); + } + + let mut fallback_keys = HashSet::new(); + for candidate in normalized_candidates { + if let Some(prefix) = candidate_prefix(candidate) { + if let Some(keys) = self.prefixes.get(&prefix) { + fallback_keys.extend(keys.iter().cloned()); } } } - - pre.iter() - .filter_map(|(candidate, code)| { - self.phonetics.get(code).and_then(|keys| { - keys.iter() - .map(|norm_key| { - let jw = strsim::jaro_winkler(norm_key, candidate); - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let score = ((jw * 100.0).round() as u16).min(100); - (score, norm_key) - }) - .max_by_key(|(score, _)| *score) - }) - }) - .max_by_key(|(score, _)| *score) - .and_then(|(score, key)| (score >= match_threshold).then(|| key.clone())) + self.select_unambiguous_candidate( + normalized_candidates, + fallback_keys.iter(), + self.smart_match_config.best_match_threshold, + guide_country, + ) } + + fn select_unambiguous_candidate<'a, I>( + &self, + guide_candidates: &[Arc], + playlist_candidates: I, + threshold: u16, + guide_country: Option<&str>, + ) -> Option + where + I: IntoIterator>, + { + let mut best: Option = None; + let mut runner_up: Option = None; + for playlist_key in playlist_candidates { + let (_, playlist_country) = self.split_country(playlist_key); + let Some(score) = guide_candidates + .iter() + .filter(|guide_key| { + numeric_signature_matches(playlist_key, guide_key) + && countries_compatible(playlist_country, guide_country) + && self.normalized_countries_compatible(playlist_key, guide_key) + }) + .map(|guide_key| similarity_score(playlist_key, guide_key)) + .max() + else { + continue; + }; + let candidate = FuzzyCandidate { key: Arc::clone(playlist_key), score }; + if best.as_ref().is_none_or(|current| candidate_precedes(&candidate, current)) { + runner_up = best.replace(candidate); + } else if runner_up.as_ref().is_none_or(|current| candidate_precedes(&candidate, current)) { + runner_up = Some(candidate); + } + } + + let best = best?; + if best.score < threshold { + return None; + } + if let Some(runner_up) = runner_up { + if best.score == runner_up.score { + return None; + } + if best.score < self.smart_match_config.best_match_threshold + && best.score.saturating_sub(runner_up.score) < MIN_FUZZY_SCORE_MARGIN + { + return None; + } + } + Some(best) + } + + fn split_country<'a>(&self, value: &'a str) -> (&'a str, Option<&'a str>) { + let split = match &self.smart_match_config.name_prefix { + EpgNamePrefix::Suffix(separator) if !separator.is_empty() => value.rsplit_once(separator), + EpgNamePrefix::Prefix(separator) if !separator.is_empty() => { + value.split_once(separator).map(|(country, core)| (core, country)) + } + EpgNamePrefix::Ignore | EpgNamePrefix::Suffix(_) | EpgNamePrefix::Prefix(_) => None, + }; + split + .filter(|(_, country)| CONSTANTS.country_codes.contains(country)) + .map_or((value, None), |(core, country)| (core, Some(country))) + } + + fn common_country<'a>(&self, candidates: &'a [Arc]) -> Option<&'a str> { + let mut country = None; + for candidate in candidates { + let (_, candidate_country) = self.split_country(candidate); + if let Some(candidate_country) = candidate_country { + if country.is_some_and(|current| current != candidate_country) { + return None; + } + country = Some(candidate_country); + } + } + country + } + + fn normalized_countries_compatible(&self, left: &str, right: &str) -> bool { + let (_, left_country) = self.split_country(left); + let (_, right_country) = self.split_country(right); + countries_compatible(left_country, right_country) + } + + fn normalized_id_name_compatible(&self, folded_epg_id: &str, playlist_name: &str) -> Option { + let guide_names = self.guide_names.get(folded_epg_id)?; + let (playlist_core, playlist_country) = self.split_country(playlist_name); + let best_score = guide_names + .iter() + .filter_map(|guide_name| { + let (guide_core, guide_country) = self.split_country(guide_name); + (countries_compatible(playlist_country, guide_country) + && numeric_signature_matches(playlist_core, guide_core)) + .then(|| similarity_score(playlist_core, guide_core)) + }) + .max(); + Some(best_score.is_some_and(|score| score >= self.smart_match_config.match_threshold)) + } + + fn channel_name_matches_epg_id(&self, epg_id: &str, channel_name: &str) -> Option { + let folded_id = with_folded_epg_id(epg_id, |folded| folded.intern()); + let normalized_name = self.normalize(channel_name); + self.normalized_id_name_compatible(&folded_id, &normalized_name) + } + + fn matched_epg_id(&self, key: &str) -> Option<(Arc, SmartMatchKind)> { + let normalized = self.normalize(key).intern(); + let epg_id = self.normalized.get(&normalized)?.as_ref()?; + let kind = self.match_ranks.get(&normalized)?.kind; + Some((Arc::clone(epg_id), kind)) + } +} + +fn candidate_precedes(left: &FuzzyCandidate, right: &FuzzyCandidate) -> bool { + left.score > right.score || (left.score == right.score && left.key < right.key) +} + +fn compare_ranked_matches(left: &RankedEpgMatch, right: &RankedEpgMatch) -> Ordering { + left.rank.cmp_preference(&right.rank).then_with(|| right.epg_id.cmp(&left.epg_id)) +} + +fn best_ranked_match<'a, I>(candidates: I) -> Option<&'a RankedEpgMatch> +where + I: IntoIterator, +{ + let mut best = None; + let mut ambiguous = false; + for candidate in candidates { + let Some(current) = best else { + best = Some(candidate); + continue; + }; + match candidate.rank.cmp_preference(¤t.rank) { + Ordering::Greater => { + best = Some(candidate); + ambiguous = false; + } + Ordering::Equal if !candidate.epg_id.eq_ignore_ascii_case(¤t.epg_id) => ambiguous = true, + Ordering::Equal | Ordering::Less => {} + } + } + (!ambiguous).then_some(best).flatten() +} + +fn countries_compatible(left: Option<&str>, right: Option<&str>) -> bool { + left.zip(right).is_none_or(|(left, right)| left == right) +} + +fn is_decorative_channel_name(name: &str) -> bool { + let trimmed = name.trim(); + if trimmed.is_empty() { + return false; + } + if !trimmed.chars().any(char::is_alphanumeric) { + return true; + } + let separator = trimmed.chars().next().expect("trimmed name is not empty"); + if separator.is_alphanumeric() { + return false; + } + trimmed.chars().take_while(|&character| character == separator).count() >= 2 + && trimmed.chars().rev().take_while(|&character| character == separator).count() >= 2 +} + +fn candidate_prefix(value: &str) -> Option { + let mut prefix = 0u32; + let mut len = 0u32; + for byte in value.bytes().filter(u8::is_ascii_alphanumeric).take(3) { + prefix = (prefix << 8) | u32::from(byte.to_ascii_lowercase()); + len += 1; + } + (len > 0).then_some((len << 24) | prefix) +} + +fn numeric_signature_matches(left: &str, right: &str) -> bool { + left.bytes().filter(u8::is_ascii_digit).eq(right.bytes().filter(u8::is_ascii_digit)) +} + +fn similarity_score(left: &str, right: &str) -> u16 { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let score = (strsim::jaro_winkler(left, right) * 100.0).round() as u16; + score.min(100) +} + +#[derive(Clone, Copy, Debug)] +enum ChannelEpgAssignment { + Existing, + Exact { corrected: bool }, + Fuzzy { corrected: bool }, + Unresolved, +} + +#[derive(Debug, Default)] +struct EpgAssignmentStats { + live: usize, + existing: usize, + exact: usize, + fuzzy: usize, + corrected: usize, + unresolved: usize, +} + +impl EpgAssignmentStats { + fn record(&mut self, assignment: ChannelEpgAssignment) { + self.live += 1; + match assignment { + ChannelEpgAssignment::Existing => self.existing += 1, + ChannelEpgAssignment::Exact { corrected } => { + self.exact += 1; + self.corrected += usize::from(corrected); + } + ChannelEpgAssignment::Fuzzy { corrected } => { + self.fuzzy += 1; + self.corrected += usize::from(corrected); + } + ChannelEpgAssignment::Unresolved => self.unresolved += 1, + } + } +} + +fn assign_smart_epg_id(chan: &mut PlaylistItem, id_cache: &EpgIdCache) -> ChannelEpgAssignment { + let current_id_is_valid = + chan.header.epg_channel_id.as_deref().is_some_and(|epg_id| id_cache.contains_processed_epg_id(epg_id)); + let current_id_is_incompatible = current_id_is_valid + && chan + .header + .epg_channel_id + .as_deref() + .is_some_and(|epg_id| id_cache.channel_name_matches_epg_id(epg_id, &chan.header.name) == Some(false)); + if current_id_is_valid && !current_id_is_incompatible { + return ChannelEpgAssignment::Existing; + } + + let matched = id_cache + .matched_epg_id(&chan.header.name) + .or_else(|| chan.header.epg_channel_id.as_deref().and_then(|id| id_cache.matched_epg_id(id))); + let Some((new_id, kind)) = matched else { + return if current_id_is_valid { ChannelEpgAssignment::Existing } else { ChannelEpgAssignment::Unresolved }; + }; + let changes_id = + chan.header.epg_channel_id.as_deref().is_none_or(|current_id| !current_id.eq_ignore_ascii_case(&new_id)); + if !changes_id { + return ChannelEpgAssignment::Existing; + } + + trace!("Matched channel {} to epg {new_id:?}", chan.header.name); + chan.header.epg_channel_id = Some(new_id); + match kind { + SmartMatchKind::Exact => ChannelEpgAssignment::Exact { corrected: current_id_is_valid }, + SmartMatchKind::Fuzzy => ChannelEpgAssignment::Fuzzy { corrected: current_id_is_valid }, + } +} + +fn assign_epg_icon( + chan: &mut PlaylistItem, + icon_tags: &HashMap, &Arc>, + icon_override_channels: &HashSet>, + icon_assigned: &mut HashSet>, +) { + let Some(epg_channel_id) = chan.header.epg_channel_id.as_ref() else { + return; + }; + with_folded_epg_id(epg_channel_id, |folded_id| { + let needs_icon = icon_override_channels.contains(folded_id) + || chan.header.logo.is_empty() + || chan.header.logo_small.is_empty(); + if icon_assigned.contains(folded_id) || !needs_icon { + return; + } + let Some(icon) = icon_tags.get(folded_id) else { + return; + }; + icon_assigned.insert(folded_id.intern()); + if icon_override_channels.contains(folded_id) || chan.header.logo.is_empty() { + trace!("Matched channel {} to epg icon {icon}", chan.header.name); + chan.header.logo = Arc::clone(icon); + } + if icon_override_channels.contains(folded_id) || chan.header.logo_small.is_empty() { + chan.header.logo_small = Arc::clone(icon); + } + }); +} + +fn referenced_live_epg_ids(fp: &mut FetchedPlaylist<'_>) -> HashSet> { + fp.items() + .filter(|channel| channel.header.xtream_cluster == XtreamCluster::Live && channel.header.item_type.is_live()) + .filter_map(|channel| { + channel.header.epg_channel_id.as_ref().map(|id| with_folded_epg_id(id, |folded| folded.intern())) + }) + .collect() } /// Assigns EPG IDs and logos to live playlist channels by matching them with EPG data. @@ -259,72 +746,61 @@ impl EpgIdCache { /// assign_channel_epg(&mut new_epg, &mut playlist, &mut id_cache); /// ``` async fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_>, id_cache: &mut EpgIdCache) { - //id_cache.normalized.retain(|_, v| v.is_some()); if let Some(tv_guide) = &fp.epg { - if let Some((epg_source, icon_override_channels)) = tv_guide.filter_merged_with_icon_overrides(id_cache).await { - let mut icon_assigned = HashSet::new(); - let icon_tags: HashMap, &Arc> = epg_source.children - .iter() - .filter_map(|tag| { - tag.icon.as_ref().filter(|icon| !icon.is_empty()).map(|icon| { - (with_folded_epg_id(&tag.id, |folded| folded.intern()), icon) + if let Some((mut epg_source, icon_override_channels)) = + tv_guide.filter_merged_with_icon_overrides(id_cache).await + { + let stats = { + let icon_tags = epg_source + .children + .iter() + .filter_map(|tag| { + tag.icon + .as_ref() + .filter(|icon| !icon.is_empty()) + .map(|icon| (with_folded_epg_id(&tag.id, |folded| folded.intern()), icon)) }) - }) - .collect(); - let icon_override_channels = icon_override_channels - .into_iter() - .map(|id| with_folded_epg_id(&id, |folded| folded.intern())) - .collect::>(); + .collect::, &Arc>>(); + let icon_override_channels = icon_override_channels + .into_iter() + .map(|id| with_folded_epg_id(&id, |folded| folded.intern())) + .collect::>(); + let mut icon_assigned = HashSet::new(); + let mut stats = EpgAssignmentStats::default(); - let assign_values = |chan: &mut PlaylistItem| { - if id_cache.smart_match_enabled { - // id_cache.processed contains all epg_ids found in any xmltv source. - let not_found_in_epg = match &chan.header.epg_channel_id { - None => true, - Some(epg_id) => !id_cache.contains_processed_epg_id(epg_id), - }; - if not_found_in_epg { - let try_match = |key: &str| { - let normalized = id_cache.normalize(key).intern(); - id_cache.normalized.get(&normalized).and_then(|epg_id| { - epg_id.as_ref().map(|id| { - trace!("Matched channel {} to epg {id:?}", chan.header.name); - id.clone() - }) - }) - }; - if let Some(new_id) = - try_match(&chan.header.name).or_else(|| chan.header.epg_channel_id.as_deref().and_then(try_match)) - { - chan.header.epg_channel_id = Some(new_id); - } - } - } - if let Some(epg_channel_id) = chan.header.epg_channel_id.as_ref() { - with_folded_epg_id(epg_channel_id, |folded_epg_channel_id| { - if !icon_assigned.contains(folded_epg_channel_id) && - (icon_override_channels.contains(folded_epg_channel_id) || chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) { - if let Some(icon) = icon_tags.get(folded_epg_channel_id) { - icon_assigned.insert(folded_epg_channel_id.intern()); - if icon_override_channels.contains(folded_epg_channel_id) || chan.header.logo.is_empty() { - trace!("Matched channel {} to epg icon {icon}", chan.header.name); - chan.header.logo = Arc::clone(icon); + if fp.is_memory() { + fp.items_mut() + .filter(|channel| { + channel.header.xtream_cluster == XtreamCluster::Live && channel.header.item_type.is_live() + }) + .for_each(|channel| { + if id_cache.smart_match_enabled { + stats.record(assign_smart_epg_id(channel, id_cache)); } - if icon_override_channels.contains(folded_epg_channel_id) || chan.header.logo_small.is_empty() { - chan.header.logo_small = Arc::clone(icon); - } - } - } - }); + assign_epg_icon(channel, &icon_tags, &icon_override_channels, &mut icon_assigned); + }); + } else { + warn!("Disk based playlist modification is not supported!"); } + stats }; - let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live && c.header.item_type.is_live(); + let referenced_epg_ids = referenced_live_epg_ids(fp); + epg_source + .children + .retain(|channel| with_folded_epg_id(&channel.id, |folded| referenced_epg_ids.contains(folded))); - if fp.is_memory() { - fp.items_mut().filter(filter_live).for_each(assign_values); - } else { - warn!("Disk based playlist modification is not supported!"); + if id_cache.smart_match_enabled { + debug!( + "Smart EPG summary for input '{}': live={}, existing={}, exact={}, fuzzy={}, corrected={}, unresolved={}", + fp.input.name, + stats.live, + stats.existing, + stats.exact, + stats.fuzzy, + stats.corrected, + stats.unresolved + ); } new_epg.push(epg_source); @@ -358,30 +834,26 @@ pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec String { - rand::rng() - .sample_iter(&Alphanumeric) - .take(30) - .map(char::from) - .collect() - } + fn random_string() -> String { rand::rng().sample_iter(&Alphanumeric).take(30).map(char::from).collect() } fn write_ics_file(path: &std::path::Path) { fs::write( @@ -417,6 +889,230 @@ mod tests { } } + fn smart_cache(match_threshold: u16, best_match_threshold: u16) -> super::EpgIdCache { + let mut dto = shared::model::EpgSmartMatchConfigDto { + enabled: true, + fuzzy_matching: true, + match_threshold, + best_match_threshold, + ..shared::model::EpgSmartMatchConfigDto::default() + }; + dto.prepare().expect("valid smart-match config"); + let config = EpgConfig { sources: vec![], smart_match: Some(EpgSmartMatchConfig::from(dto)) }; + super::EpgIdCache::new(Some(&config)) + } + + fn smart_cache_with_country_suffix() -> super::EpgIdCache { + let mut dto = shared::model::EpgSmartMatchConfigDto { + enabled: true, + fuzzy_matching: true, + name_prefix: shared::model::EpgNamePrefix::Suffix(".".to_string()), + ..shared::model::EpgSmartMatchConfigDto::default() + }; + dto.prepare().expect("valid smart-match config"); + let config = EpgConfig { sources: vec![], smart_match: Some(EpgSmartMatchConfig::from(dto)) }; + super::EpgIdCache::new(Some(&config)) + } + + async fn run_smart_xmltv_match( + xmltv: &str, + channel_name: &str, + current_epg_id: Option<&str>, + ) -> (Option>, Vec) { + let (mut assigned_ids, epg) = run_smart_xmltv_matches(xmltv, &[(channel_name, current_epg_id)]).await; + (assigned_ids.pop().flatten(), epg) + } + + async fn run_smart_xmltv_matches( + xmltv: &str, + channels: &[(&str, Option<&str>)], + ) -> (Vec>>, Vec) { + run_xmltv_matches(xmltv, channels, true).await + } + + async fn run_xmltv_matches( + xmltv: &str, + channels: &[(&str, Option<&str>)], + smart_matching: bool, + ) -> (Vec>>, Vec) { + let dir = tempdir().unwrap(); + let epg_path = dir.path().join("smart-match.xml"); + fs::write(&epg_path, xmltv).unwrap(); + + let smart_match = smart_matching.then(|| { + let mut dto = shared::model::EpgSmartMatchConfigDto { + enabled: true, + fuzzy_matching: true, + ..shared::model::EpgSmartMatchConfigDto::default() + }; + dto.prepare().expect("smart config"); + EpgSmartMatchConfig::from(dto) + }); + let mut input = ConfigInput::from(ConfigInputDto::default()); + input.epg = Some(EpgConfig { sources: vec![], smart_match }); + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: channels.iter().map(|(name, epg_id)| live_playlist_item(name, *epg_id)).collect(), + xtream_cluster: super::XtreamCluster::Live, + }]; + let tv_guide = TVGuide::new(vec![PersistedEpgSource { + file_path: epg_path, + priority: 0, + logo_override: false, + kind: PersistedEpgSourceKind::Xmltv, + }]); + let mut playlist = FetchedPlaylist { + input: &input, + source: MemoryPlaylistSource::new(groups).into_source(), + epg: Some(tv_guide), + }; + let mut epg = Vec::new(); + + super::process_playlist_epg(&mut playlist, &mut epg).await; + let assigned_ids = playlist.items_mut().map(|item| item.header.epg_channel_id.clone()).collect(); + (assigned_ids, epg) + } + + #[test] + fn smart_match_prefers_exact_matches_from_higher_priority_sources() { + let mut cache = smart_cache(80, 95); + cache.insert_normalized("TF1", None); + let candidates = cache.normalize_candidates(["TF1.fr", "TF1"]); + + assert!(cache.match_epg_channel_candidates(&"tf1.low".intern(), &candidates, 10, 1)); + assert!(cache.match_epg_channel_candidates(&"tf1.high".intern(), &candidates, 0, 2)); + assert!(cache.match_epg_channel_candidates(&"tf1.lower".intern(), &candidates, 20, 0)); + + cache.finalize_matches(&HashSet::from(["tf1.low".intern(), "tf1.high".intern(), "tf1.lower".intern()])); + assert_eq!(cache.normalized.get("tf1").and_then(Option::as_deref), Some("tf1.high")); + } + + #[test] + fn smart_match_replaces_an_earlier_fuzzy_candidate_with_a_better_score() { + let mut cache = smart_cache(70, 70); + cache.insert_normalized("RMC Decouverte", None); + let key = cache.normalize("RMC Decouverte").intern(); + let weaker = cache.normalize_candidates(["RMC Decouver"]); + let stronger = cache.normalize_candidates(["RMC Decouvert"]); + assert!(super::similarity_score(&key, &weaker[0]) < super::similarity_score(&key, &stronger[0])); + + assert!(cache.match_epg_channel_candidates(&"rmc.weak".intern(), &weaker, 0, 0)); + assert!(cache.match_epg_channel_candidates(&"rmc.strong".intern(), &stronger, 0, 0)); + assert_eq!(cache.normalized.get(&key).and_then(Option::as_deref), Some("rmc.strong")); + } + + #[test] + fn smart_match_rejects_conflicting_channel_numbers() { + let mut cache = smart_cache(70, 70); + cache.insert_normalized("TF1", None); + let candidates = cache.normalize_candidates(["TF1+1"]); + + assert!(!cache.match_epg_channel_candidates(&"tf1-plus-one.fr".intern(), &candidates, 0, 0)); + assert_eq!(cache.normalized.get("tf1").and_then(Option::as_deref), None); + } + + #[test] + fn smart_match_rejects_ambiguous_equal_scores() { + let cache = smart_cache(70, 95); + let guide = vec!["abcc".intern()]; + let left = "abca".intern(); + let right = "abcb".intern(); + assert_eq!(super::similarity_score(&left, &guide[0]), super::similarity_score(&right, &guide[0])); + + assert!(cache.select_unambiguous_candidate(&guide, [&left, &right], 70, None).is_none()); + } + + #[test] + fn smart_match_country_fallback_prefers_the_full_country_match() { + let mut cache = smart_cache_with_country_suffix(); + cache.insert_normalized("FR|TF1", None); + let french = cache.normalize_candidates(["TF1.fr", "TF1"]); + let belgian = cache.normalize_candidates(["TF1.be", "TF1"]); + assert!(cache.match_epg_channel_candidates(&"TF1.fr".intern(), &french, 0, 0)); + assert!(!cache.match_epg_channel_candidates(&"TF1.be".intern(), &belgian, 0, 0)); + + cache.finalize_matches(&HashSet::from(["tf1.fr".intern(), "tf1.be".intern()])); + assert_eq!(cache.normalized.get("tf1.fr").and_then(Option::as_deref), Some("TF1.fr")); + } + + #[test] + fn smart_match_rejects_a_display_country_conflicting_with_the_epg_id() { + let mut cache = smart_cache_with_country_suffix(); + cache.insert_normalized("FR|COMEDIE+", None); + let candidates = cache.normalize_candidates(["ComediePlus.mu", "FR|COMEDIE+"]); + + assert!(!cache.match_epg_channel_candidates(&"ComediePlus.mu".intern(), &candidates, 0, 0)); + assert_eq!(cache.normalized.get("comedie.fr").and_then(Option::as_deref), None); + } + + #[test] + fn smart_match_rejects_ambiguous_countryless_fallbacks() { + let mut cache = smart_cache_with_country_suffix(); + cache.insert_normalized("FR|TF1", None); + let display_name = cache.normalize_candidates(["TF1"]); + assert!(cache.match_epg_channel_candidates(&"opaque-a".intern(), &display_name, 0, 0)); + assert!(cache.match_epg_channel_candidates(&"opaque-b".intern(), &display_name, 0, 0)); + + cache.finalize_matches(&HashSet::from(["opaque-a".intern(), "opaque-b".intern()])); + assert_eq!(cache.normalized.get("tf1.fr").and_then(Option::as_deref), None); + } + + #[test] + fn selected_epg_ids_exclude_unused_smart_matches() { + let mut cache = smart_cache(80, 95); + let valid_id = "tf1.fr".intern(); + cache.insert_channel_epg_id(&valid_id); + cache.normalize_and_store("TF1", Some(&valid_id)); + cache.normalize_and_store("France 2", None); + + let tf1_candidates = cache.normalize_candidates(["TF1"]); + let france2_candidates = cache.normalize_candidates(["France 2"]); + assert!(cache.match_epg_channel_candidates(&"tf1.unused".intern(), &tf1_candidates, 0, 0)); + assert!(cache.match_epg_channel_candidates(&"france2.fr".intern(), &france2_candidates, 0, 0)); + + let available = HashSet::from([valid_id.clone(), "tf1.unused".intern(), "france2.fr".intern()]); + cache.finalize_matches(&available); + let selected = cache.selected_epg_ids(&available); + assert!(selected.contains(&valid_id)); + assert!(selected.contains("france2.fr")); + assert!(!selected.contains("tf1.unused")); + } + + #[test] + fn guide_names_ignore_empty_normalized_values_and_unreferenced_ids() { + let mut cache = smart_cache(80, 95); + cache.insert_channel_epg_id("direct.fr"); + + assert!(cache.needs_guide_names("DIRECT.FR")); + assert!(!cache.needs_guide_names("unreferenced.fr")); + cache.register_guide_names("direct.fr", [" "]); + + assert_eq!(cache.channel_name_matches_epg_id("direct.fr", "Direct"), None); + } + + #[test] + fn smart_match_ignores_decorative_playlist_entries() { + let mut cache = smart_cache(80, 95); + let input = ConfigInput::from(ConfigInputDto::default()); + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![live_playlist_item("▬▬ Manga ▬▬", None)], + xtream_cluster: super::XtreamCluster::Live, + }]; + let mut playlist = + FetchedPlaylist { input: &input, source: MemoryPlaylistSource::new(groups).into_source(), epg: None }; + + cache.collect_epg_id(&mut playlist); + + assert!(cache.normalized.is_empty()); + assert!(super::is_decorative_channel_name("▬▬ Manga ▬▬")); + assert!(super::is_decorative_channel_name("----")); + assert!(!super::is_decorative_channel_name("|Channel|")); + assert!(!super::is_decorative_channel_name("")); + } + #[test] fn channel_epg_id_membership_is_case_insensitive_ascii() { let mut cache = super::EpgIdCache::new(None); @@ -497,6 +1193,96 @@ mod tests { }); } + #[test] + fn smart_match_replaces_an_existing_id_that_has_no_programmes() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let (assigned_id, epg) = run_smart_xmltv_match( + r#" + Unused feed + TF1 + + Programme TF1 + +"#, + "TF1", + Some("empty.fr"), + ) + .await; + + assert_eq!(assigned_id.as_deref(), Some("tf1.fr")); + assert_eq!(epg[0].children.len(), 1); + assert_eq!(epg[0].children[0].id.as_ref(), "tf1.fr"); + assert_eq!(epg[0].children[0].programmes.len(), 1); + }); + } + + #[test] + fn directly_referenced_empty_guides_are_retained_with_or_without_smart_matching() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let xmltv = r#" + Empty channel +"#; + for smart_matching in [false, true] { + let (assigned_ids, epg) = + run_xmltv_matches(xmltv, &[("Empty channel", Some("empty.fr"))], smart_matching).await; + + assert_eq!(assigned_ids, vec![Some("empty.fr".intern())]); + assert_eq!(epg[0].children.len(), 1); + assert_eq!(epg[0].children[0].id.as_ref(), "empty.fr"); + assert!(epg[0].children[0].programmes.is_empty()); + } + }); + } + + #[test] + fn smart_match_assigns_variants_when_a_direct_id_is_already_known() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let (assigned_ids, epg) = run_smart_xmltv_matches( + r#" + TMC HD + + Programme TMC + +"#, + &[("TMC HD", Some("tmc.fr")), ("TMC FHD", None), ("TMC SD", None)], + ) + .await; + + assert_eq!(assigned_ids, vec![Some("tmc.fr".intern()); 3]); + assert_eq!(epg[0].children.len(), 1); + assert_eq!(epg[0].children[0].programmes.len(), 1); + }); + } + + #[test] + fn smart_match_corrects_a_populated_but_semantically_wrong_id() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let (assigned_id, epg) = run_smart_xmltv_match( + r#" + Cherie 25 + RMC Life + + Programme Cherie 25 + + + Programme RMC Life + +"#, + "RMC LIFE", + Some("cherie25.fr"), + ) + .await; + + assert_eq!(assigned_id.as_deref(), Some("rmclife.fr")); + assert_eq!(epg[0].children.len(), 1); + assert_eq!(epg[0].children[0].id.as_ref(), "rmclife.fr"); + }); + } + #[test] fn playlist_item_with_ics_epg_channel_id_gets_ics_programmes() { let runtime = tokio::runtime::Runtime::new().unwrap(); @@ -544,10 +1330,7 @@ mod tests { }; smart_dto.prepare().expect("smart config"); let mut input = ConfigInput::from(ConfigInputDto::default()); - input.epg = Some(EpgConfig { - sources: vec![], - smart_match: Some(EpgSmartMatchConfig::from(smart_dto)), - }); + input.epg = Some(EpgConfig { sources: vec![], smart_match: Some(EpgSmartMatchConfig::from(smart_dto)) }); let groups = vec![PlaylistGroup { id: 1, title: "Live".intern(), @@ -572,9 +1355,7 @@ mod tests { #[test] fn test_phonetic() { - let strings: Vec = (0..5_000) - .map(|_| random_string()) - .collect(); + let strings: Vec = (0..5_000).map(|_| random_string()).collect(); let phonetic = DoubleMetaphone::new(Some(6)); diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index 4b3bf0b52..9d1e1580f 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -682,11 +682,14 @@ For ICS sources, Tuliprox uses the generated virtual channel metadata: | `enabled` | Bool | `false` | Activates the Smart Match engine for streams without a fixed `tvg-id`. | | `fuzzy_matching` | Bool | `false` | Fallback to phonetic and Jaro-Winkler similarity matching if exact ID match fails. | | `match_threshold` | Int | `80` | Minimum similarity score (10-100) required to accept a fuzzy match. | -| `best_match_threshold` | Int | `99` | Score at which Tuliprox stops searching and immediately accepts the EPG assignment. | +| `best_match_threshold` | Int | `95` | Minimum score for the strict fallback used when phonetic keys differ. | | `name_prefix` | Object | `ignore` | Options: `ignore`, `suffix`, `prefix`. For `suffix`/`prefix`, a concat string (e.g., `{ suffix: "." }`) is required. | | `name_prefix_separator` | List | `[':', '\|', '-']` | Characters used by providers to delimit country codes (e.g., `US:`, `FR\|`). | -| `strip` | List | *(HD/4K tags)* | Default: `["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]`. Terms stripped before matching. | -| `normalize_regex` | String | `[^a-zA-Z0-9\-]` | Default pattern to strip non-alphanumeric characters (except dashes) for cleaner matching. | +| `strip` | List | *(quality tags)* | Resolution, codec and frame-rate markers stripped as complete terms before matching. | +| `normalize_regex` | String | `[^a-zA-Z0-9._\-]` | Default pattern preserving the separators commonly found in XMLTV channel IDs. | + +When upgrading, an explicitly configured legacy pattern such as `[^a-zA-Z0-9\-]` remains unchanged and continues to +remove periods and underscores. Remove that override or set `[^a-zA-Z0-9._\-]` to adopt the new default behavior. #### How Smart-Matching works @@ -699,8 +702,21 @@ If a stream is missing the `tvg-id`, Tuliprox performs the following steps: becomes `hbo`. 4. **Reconstruction:** Using `name_prefix.suffix` (`.`), the country code is appended to the name. The target search key becomes `hbo.us`. -5. **Phonetic Matching:** The engine uses **Double Metaphone** phonetic encoding to search for the ID `hbo.us` in the - aggregated EPG database. +5. **Exact Matching:** Normalized XMLTV IDs and display names are checked first. Source priority resolves duplicate + exact candidates. Quality variants sharing one normalized name can reuse a populated direct ID. +6. **Fuzzy Matching:** When enabled, the engine scores every candidate in the matching **Double Metaphone** bucket and + keeps the globally best result rather than the first acceptable result. If the phonetic lookup yields nothing, a + same-initial fallback is allowed only at `best_match_threshold`. +7. **Safety Checks:** Numeric signatures must agree (`TF1` cannot match `TF1+1`), tied candidates are rejected, and + low-confidence candidates must beat the runner-up by a minimum margin. Explicit XMLTV ID country suffixes prevent + cross-country matches, and decorative playlist separators are ignored. +8. **Programme Validation:** A channel declaration without programmes is not treated as a successful guide match. + Tuliprox keeps ranked alternatives while parsing and selects the best candidate that actually contributes programme + data. Existing IDs with no programmes can therefore be replaced by a populated, semantically compatible candidate. + ICS channels with an enabled dummy policy also count as populated. + +At `debug` level, each processed input emits one `Smart EPG summary` with the number of live channels whose original ID +was valid, whose ID was assigned by Smart Match, or which remained unresolved. For an ICS source, the generated EPG channel participates in the same matching process. For example, this configuration: diff --git a/shared/src/defaults/epg.rs b/shared/src/defaults/epg.rs index a39d6081d..90f6b6943 100644 --- a/shared/src/defaults/epg.rs +++ b/shared/src/defaults/epg.rs @@ -42,7 +42,9 @@ pub fn is_default_ics_event_description(value: &String) -> bool { value == DEFAU pub fn default_ics_dummy_title() -> String { DEFAULT_ICS_DUMMY_TITLE.to_string() } pub fn is_default_ics_dummy_title(value: &String) -> bool { value == DEFAULT_ICS_DUMMY_TITLE } -pub const DEFAULT_EPG_NORMALIZE_REGEX: &str = r"[^a-zA-Z0-9\-]"; +/// Preserves `.`, `_`, and `-` so normalized names remain comparable with common XMLTV IDs. +/// The former `[^a-zA-Z0-9\-]` pattern remains valid only as an explicit legacy configuration. +pub const DEFAULT_EPG_NORMALIZE_REGEX: &str = r"[^a-zA-Z0-9._\-]"; pub fn default_epg_normalize_regex() -> Option { Some(DEFAULT_EPG_NORMALIZE_REGEX.to_string()) } pub fn is_default_epg_normalize_regex(v: &Option) -> bool { @@ -52,7 +54,10 @@ pub fn is_default_epg_normalize_regex(v: &Option) -> bool { } } -pub const DEFAULT_EPG_STRIP: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw", "full hd"]; +pub const DEFAULT_EPG_STRIP: &[&str] = &[ + "3840p", "2160p", "1080p", "720p", "576p", "uhd", "fhd", "full hd", "hd", "sd", "4k", "h265", "h264", "hevc", + "50fps", "60fps", "plus", "raw", +]; pub const DEFAULT_EPG_NAME_PREFIX_SEPARATOR: &[char] = &[':', '|', '-']; pub fn default_epg_strip() -> Option> {