diff --git a/src/model/config.rs b/src/model/config.rs index 945647de9..1754aa46f 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -1003,6 +1003,13 @@ pub struct EpgSmartMatchConfig { } impl EpgSmartMatchConfig { + + pub fn new() -> Result { + let mut this = Self { enabled: true, ..Self::default() }; + this.prepare()?; + Ok(this) + } + /// # Panics pub fn prepare(&mut self) -> Result<(), M3uFilterError> { if !self.enabled { diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index 54dff5891..eeb41ac6a 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -1,6 +1,8 @@ use crate::model::config::{EpgNamePrefix, EpgSmartMatchConfig}; use crate::model::xmltv::{Epg, TVGuide, XmlTag, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_CHANNEL, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_PROGRAMME, EPG_TAG_TV}; +use crate::processing::processor::epg::EpgIdCache; use crate::utils::compression::compressed_file_reader::CompressedFileReader; +use crate::utils::constants::CONSTANTS; use deunicode::deunicode; use quick_xml::events::{BytesStart, BytesText, Event}; use quick_xml::Reader; @@ -10,10 +12,8 @@ use std::cmp::min; use std::collections::{HashMap, HashSet}; use std::mem; use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU16, Ordering}; use std::sync::{Arc, Mutex, RwLock}; -use crate::processing::processor::epg::EpgIdCache; -use crate::utils::constants::CONSTANTS; fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a str>, &'a str) { for delim in delimiters { @@ -126,39 +126,38 @@ impl TVGuide { fn find_best_fuzzy_match(id_cache: &mut EpgIdCache, tag: &XmlTag) -> (bool, Option) { let early_exit_flag = Arc::new(AtomicBool::new(false)); // Flag für den frühen Abbruch - let data: Mutex<(u16, Option>)> = Mutex::new((0, None)); + let data: Mutex>> = Mutex::new(None); + let current_best_match = AtomicU16::new(0); // Flag für den frühen Abbruch let match_threshold = id_cache.smart_match_config.match_threshold; let best_match_threshold = id_cache.smart_match_config.best_match_threshold; - id_cache.normalized.par_iter().for_each(|(norm_key, _epg_id_opt)| { - if early_exit_flag.load(Ordering::SeqCst) { - return; - } - let phonetic_code = id_cache.phonetic(norm_key); - for normalized_epg_id in &tag.normalized_epg_ids { - let code = id_cache.phonetic(normalized_epg_id); - if code == phonetic_code { - let match_jw = strsim::jaro_winkler(norm_key, normalized_epg_id); + for tag_normalized in &tag.normalized_epg_ids { + let tag_code = id_cache.phonetic(tag_normalized); + if let Some(normalized) = id_cache.phonetics.get(&tag_code) { + normalized.par_iter().find_any(|norm_key| { + let match_jw = strsim::jaro_winkler(norm_key, tag_normalized); #[allow(clippy::cast_possible_truncation)] #[allow(clippy::cast_sign_loss)] let mjw = min(100, (match_jw * 100.0).round() as u16); if mjw >= match_threshold { - let mut lock = data.lock().unwrap(); - if lock.0 < mjw { - *lock = (mjw, Some(Cow::Borrowed(norm_key))); + if current_best_match.load(Ordering::SeqCst) < mjw { + let mut lock = data.lock().unwrap(); + *lock = Some(Cow::Borrowed(norm_key)); + current_best_match.store(mjw, Ordering::SeqCst); } if mjw > best_match_threshold { - early_exit_flag.store(true, Ordering::SeqCst); - return; // (true, matched_normalized_epg_id.map(|s| s.to_string())); + return true; // (true, matched_normalized_epg_id.map(|s| s.to_string())); } } - } + false + }); } - // is there an early exit strategy ??? - }); + } + // is there an early exit strategy ??? + if early_exit_flag.load(Ordering::SeqCst) { - let result = data.lock().unwrap().1.take(); + let result = data.lock().unwrap().take(); return (true, result.as_ref().map(std::string::ToString::to_string)); } (false, None) @@ -174,25 +173,26 @@ impl TVGuide { let mut filter_tags = |mut tag: XmlTag| { match tag.name.as_str() { EPG_TAG_CHANNEL => { - Self::prepare_tag(id_cache, &mut tag, smart_match); - if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { - if !id_cache.processed.contains(epg_id) { - if smart_match { - if Self::try_fuzzy_matching(id_cache, epg_id, &tag, fuzzy_matching) { - children.push(tag); - } - } else { - let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); - if id_cache.channel_epg_id.contains(&borrowed_epg_id) { - children.push(tag); - } + let epg_id = tag.get_attribute_value(EPG_ATTRIB_ID).map_or_else(String::new, std::string::ToString::to_string); + if !epg_id.is_empty() && !id_cache.processed.contains(&epg_id) { + Self::prepare_tag(id_cache, &mut tag, smart_match); + if smart_match { + if Self::try_fuzzy_matching(id_cache, &epg_id, &tag, fuzzy_matching) { + children.push(tag); + id_cache.processed.insert(epg_id); + } + } else { + let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); + if id_cache.channel_epg_id.contains(&borrowed_epg_id) { + children.push(tag); + id_cache.processed.insert(epg_id); } } } } EPG_TAG_PROGRAMME => { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_CHANNEL) { - if !id_cache.processed.contains(epg_id) { + if id_cache.processed.contains(epg_id) { let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); if id_cache.channel_epg_id.contains(&borrowed_epg_id) { children.push(tag); @@ -213,12 +213,6 @@ impl TVGuide { return None; } - children.iter().filter(|tag| tag.name == EPG_TAG_CHANNEL).for_each(|tag| { - if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { - id_cache.processed.insert(epg_id.to_string()); - } - }); - Some(Epg { attributes: tv_attributes, children, @@ -403,6 +397,15 @@ mod tests { use crate::model::config::{EpgNamePrefix, EpgSmartMatchConfig}; use crate::processing::parser::xmltv::normalize_channel_name; + #[test] + fn parse_normalize() -> Result<(), M3uFilterError> { + let epg_normalize = EpgSmartMatchConfig::new()?; + let normalized = normalize_channel_name("Love Nature", &epg_normalize); + assert_eq!(normalized, "lovenature".to_string()); + Ok(()) + } + + // #[test] // fn parse_test() -> io::Result<()> { // let file_path = PathBuf::from("/tmp/epg.xml.gz"); @@ -437,7 +440,9 @@ mod tests { assert_eq!("odisea.bg", normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); } + use crate::m3u_filter_error::M3uFilterError; use rphonetic::{Encoder, Metaphone}; + #[test] fn test_metaphone() { let metaphone = Metaphone::default(); diff --git a/src/processing/processor/epg.rs b/src/processing/processor/epg.rs index 746ae1769..2e95eb8af 100644 --- a/src/processing/processor/epg.rs +++ b/src/processing/processor/epg.rs @@ -1,6 +1,6 @@ use crate::model::config::{EpgConfig, EpgSmartMatchConfig}; use crate::model::playlist::{FetchedPlaylist, PlaylistItem, XtreamCluster}; -use crate::model::xmltv::Epg; +use crate::model::xmltv::{Epg, XmlTag, EPG_ATTRIB_ID}; use crate::processing::parser::xmltv::normalize_channel_name; use log::debug; use rphonetic::{DoubleMetaphone, Encoder}; @@ -9,11 +9,12 @@ use std::collections::{HashMap, HashSet}; pub struct EpgIdCache<'a> { pub channel_epg_id: HashSet>, pub normalized: HashMap>, + pub phonetics: HashMap>, pub processed: HashSet, pub smart_match_config: EpgSmartMatchConfig, pub metaphone: DoubleMetaphone, - pub smart_match_enabled: bool, - pub fuzzy_match_enabled: bool, + pub smart_match_enabled: bool, // smart match is enabled, normalizing names + pub fuzzy_match_enabled: bool, // fuzzy matching enabled } impl EpgIdCache<'_> { @@ -22,6 +23,7 @@ impl EpgIdCache<'_> { EpgIdCache { channel_epg_id: HashSet::new(), // contains the epg_ids collected from playlist channels normalized: HashMap::new(), + phonetics: HashMap::new(), processed: HashSet::new(), metaphone: DoubleMetaphone::default(), smart_match_enabled: normalize_config.enabled, @@ -37,7 +39,9 @@ impl EpgIdCache<'_> { fn normalize_and_store(&mut self, name: &str, epg_id: Option<&String>) { let normalized_name = self.normalize(name); + let phonetic = self.phonetic(&normalized_name); self.normalized.insert(normalized_name, epg_id.map(std::string::ToString::to_string)); + self.phonetics.entry(phonetic.to_string()).or_default().insert(phonetic); } fn normalize(&self, name: &str) -> String { @@ -89,9 +93,9 @@ fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist, id_cache if let Some(tv_guide) = &fp.epg { if let Some(epg) = tv_guide.filter(id_cache) { // // icon tags - // let icon_tags: HashMap<&String, &XmlTag> = epg.children.iter() - // .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) - // .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t)).collect(); + let icon_tags: HashMap<&String, &XmlTag> = epg.children.iter() + .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) + .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t)).collect(); let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live; // let filter_missing_epg_id = |chan: &mut PlaylistItem| chan.header.epg_channel_id.is_none() || chan.header.logo.is_empty() || chan.header.logo_small.is_empty(); @@ -111,18 +115,18 @@ fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist, id_cache } } } - // if chan.header.epg_channel_id.is_some() && (chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) { - // if let Some(icon_tag) = icon_tags.get(chan.header.epg_channel_id.as_ref().unwrap()) { - // if let Some(icon) = icon_tag.icon.as_ref() { - // if chan.header.logo.is_empty() { - // chan.header.logo = (*icon).to_string(); - // } - // if chan.header.logo_small.is_empty() { - // chan.header.logo = (*icon).to_string(); - // } - // } - // } - // } + if chan.header.epg_channel_id.is_some() && (chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) { + if let Some(icon_tag) = icon_tags.get(chan.header.epg_channel_id.as_ref().unwrap()) { + if let Some(icon) = icon_tag.icon.as_ref() { + if chan.header.logo.is_empty() { + chan.header.logo = (*icon).to_string(); + } + if chan.header.logo_small.is_empty() { + chan.header.logo = (*icon).to_string(); + } + } + } + } }; fp.playlistgroups.iter_mut() @@ -140,7 +144,7 @@ pub fn process_playlist_epg(fp: &mut FetchedPlaylist, epg: &mut Vec) { let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); id_cache.collect_epg_id(fp); - if id_cache.is_empty() { // TODO && not fuzzy matching { + if id_cache.is_empty() && !id_cache.smart_match_enabled { debug!("No epg ids found"); } else { assign_channel_epg(epg, fp, &mut id_cache);