diff --git a/Cargo.lock b/Cargo.lock index 898923d3b..0d341f378 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -666,6 +666,21 @@ dependencies = [ "syn", ] +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -1462,6 +1477,12 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.171" @@ -1513,6 +1534,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + [[package]] name = "lock_api" version = "0.4.12" @@ -1560,6 +1587,7 @@ dependencies = [ "regex", "reqwest", "rpassword", + "rphonetic", "rust-argon2", "rustelebot", "ruzstd", @@ -1651,6 +1679,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nonempty" version = "0.7.0" @@ -2217,6 +2254,21 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rphonetic" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51016596e51b46757c38a94e15a56b293ad110fe37a6fadfea0083910c52b482" +dependencies = [ + "document-features", + "either", + "enum-iterator", + "lazy_static", + "nom", + "regex", + "serde", +] + [[package]] name = "rtoolbox" version = "0.0.2" diff --git a/Cargo.toml b/Cargo.toml index 4b5ee2c51..12ff9dbe1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ ruzstd = "0" filetime = "0.2" zeroize = "1.8" strsim = "0.11" +rphonetic = "3" #[cfg(target_os = "macos")] libc = "0" #[cfg(target_os = "windows")] diff --git a/src/api/model/active_provider_manager.rs b/src/api/model/active_provider_manager.rs index 9845d671d..c2ae7215a 100644 --- a/src/api/model/active_provider_manager.rs +++ b/src/api/model/active_provider_manager.rs @@ -528,7 +528,7 @@ mod tests { id, name: name.to_string(), url: "http://example.com".to_string(), - epg_url: None, + epg: Default::default(), username: None, password: None, persist: None, diff --git a/src/model/config.rs b/src/model/config.rs index d156e93fd..fdb021f5c 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -14,6 +14,7 @@ use crate::auth::user::UserCredential; use log::{debug, error, warn}; use path_clean::PathClean; use rand::Rng; +use regex::Regex; use url::Url; use crate::foundation::filter::{get_filter, prepare_templates, Filter, MockValueProcessor, PatternTemplate, ValueProvider}; @@ -763,6 +764,138 @@ pub enum EpgUrl { Multi(Vec) } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum EpgCountryPrefix { + #[default] + Ignore, + Suffix(String), + Prefix(String), +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EpgNormalizeConfig { + #[serde(default)] + pub enabled: bool, + pub normalize_regex: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub strip: Option>, + #[serde(default)] + pub country_prefix: EpgCountryPrefix, + #[serde(default)] + pub fuzzy_matching: bool, + #[serde(default)] + pub match_threshold: u16, + #[serde(skip)] + pub t_strip: Vec, + #[serde(skip)] + pub t_normalize_regex: Option, + #[serde(skip)] + pub t_country_prefix_separator: Vec, + #[serde(skip)] + pub t_match_threshold: f64, +} + +impl EpgNormalizeConfig { + /// # Panics + pub fn prepare(&mut self) -> Result<(), M3uFilterError> { + if !self.enabled { + return Ok(()) + } + if self.match_threshold < 70 { + warn!("match_threshold is less than 70%, setting to 70%"); + self.match_threshold = 70; + } else if self.match_threshold > 100 { + warn!("match_threshold is more than 100%, setting to 80%"); + self.match_threshold = 100; + } + self.t_match_threshold = f64::from(self.match_threshold) / 100.0; + + self.t_normalize_regex =match self.normalize_regex.as_ref() { + None => Some(Regex::new(r"[^a-zA-Z0-9\-]").unwrap()), + Some(regstr) => { + let re = regex::Regex::new(regstr.as_str()); + if re.is_err() { + return create_m3u_filter_error_result!(M3uFilterErrorKind::Info, "cant parse regex: {}", regstr); + } + Some(re.unwrap()) + } + }; + + if self.strip.is_none() { + self.t_strip = ["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"].iter().map(std::string::ToString::to_string).collect(); + } + Ok(()) + } +} + +impl Default for EpgNormalizeConfig { + fn default() -> Self { + let mut instance = EpgNormalizeConfig { + enabled: false, + normalize_regex: None, + strip: None, + country_prefix: EpgCountryPrefix::default(), + fuzzy_matching: false, + match_threshold: 0, + t_strip: Vec::default(), + t_normalize_regex: None, + t_country_prefix_separator: Vec::default(), + t_match_threshold: 0.0, + }; + let _= instance.prepare(); + instance + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EpgConfig { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub normalize: Option, + #[serde(skip)] + pub t_urls: Vec, + #[serde(skip)] + pub t_normalize: EpgNormalizeConfig, +} + +impl EpgConfig { + pub fn prepare(&mut self) -> Result<(), M3uFilterError> { + self.t_urls = self.url.take().map_or_else(Vec::new, |epg_url| { + match epg_url { + EpgUrl::Single(url) => if url.trim().is_empty() { + vec![] + } else { + vec![url.trim().to_string()] + }, + EpgUrl::Multi(urls) => + urls.into_iter() + .map(|url| url.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() + } + }); + + self.t_normalize = match self.normalize.as_mut() { + None => { + let mut normalize: EpgNormalizeConfig = EpgNormalizeConfig::default(); + normalize.prepare()?; + normalize + } + Some(normalize_cfg) => { + let mut normalize: EpgNormalizeConfig = normalize_cfg.clone(); + normalize.prepare()?; + normalize + } + }; + + Ok(()) + } +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct ConfigInput { @@ -775,9 +908,7 @@ pub struct ConfigInput { pub headers: HashMap, pub url: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub epg_url: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub epg_strip: Option>, + pub epg: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub username: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -817,20 +948,9 @@ impl ConfigInput { check_input_credentials!(self, self.input_type); self.persist = get_trimmed_string(&self.persist); - self.epg_url = self.epg_url.take().map(|epg_url| { - match epg_url { - EpgUrl::Single(url) => if url.trim().is_empty() { - EpgUrl::Multi(vec![]) - } else { - EpgUrl::Single(url.trim().to_string()) - }, - EpgUrl::Multi(urls) => - EpgUrl::Multi(urls.into_iter() - .map(|url| url.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect()) - } - }); + if let Some(epg) = self.epg.as_mut() { + let _ = epg.prepare(); + } if let Some(aliases) = self.aliases.as_mut() { let input_type = &self.input_type; diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index 354a1bee1..bcc2aafa5 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -1,29 +1,81 @@ -use std::borrow::Cow; +use crate::model::config::{EpgCountryPrefix, EpgNormalizeConfig}; use crate::model::xmltv::{Epg, TVGuide, XmlTag, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_CHANNEL, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_PROGRAMME, EPG_TAG_TV}; +use crate::processing::processor::playlist::EpgIdCache; use crate::utils::compression::compressed_file_reader::CompressedFileReader; +use deunicode::deunicode; use quick_xml::events::{BytesStart, Event}; use quick_xml::Reader; -use regex::Regex; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, LazyLock}; -use deunicode::deunicode; -use crate::processing::processor::playlist::EpgIdCache; +use rphonetic::{Encoder, Metaphone}; -static NORMALIZE_CHANNEL: LazyLock = LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9\-]").unwrap()); +static COUNTRY_CODE: LazyLock> = LazyLock::new(|| vec![ + "af", "al", "dz", "ad", "ao", "ag", "ar", "am", "au", "at", "az", "bs", "bh", "bd", "bb", "by", + "be", "bz", "bj", "bt", "bo", "ba", "bw", "br", "bn", "bg", "bf", "bi", "cv", "kh", "cm", "ca", + "cf", "td", "cl", "cn", "co", "km", "cg", "cr", "hr", "cu", "cy", "cz", "cd", "dk", "dj", "dm", + "do", "tl", "ec", "eg", "sv", "gq", "er", "ee", "sz", "et", "fj", "fi", "fr", "ga", "gm", "ge", + "de", "gh", "gr", "gd", "gt", "gn", "gw", "gy", "ht", "hn", "hu", "is", "in", "id", "ir", "iq", + "ie", "il", "it", "ci", "jm", "jp", "jo", "kz", "ke", "ki", "kp", "kr", "kw", "kg", "la", "lv", + "lb", "ls", "lr", "ly", "li", "lt", "lu", "mg", "mw", "my", "mv", "ml", "mt", "mh", "mr", "mu", + "mx", "fm", "md", "mc", "mn", "me", "ma", "mz", "mm", "na", "nr", "np", "nl", "nz", "ni", "ne", + "ng", "mk", "no", "om", "pk", "pw", "pa", "pg", "py", "pe", "ph", "pl", "pt", "qa", "ro", "ru", + "rw", "kn", "lc", "vc", "ws", "sm", "st", "sa", "sn", "rs", "sc", "sl", "sg", "sk", "si", "sb", + "so", "za", "ss", "es", "lk", "sd", "sr", "se", "ch", "sy", "tw", "tj", "tz", "th", "tg", "to", + "tt", "tn", "tr", "tm", "tv", "ug", "ua", "ae", "gb", "us", "uy", "uz", "vu", "va", "ve", "vn", + "ye", "zm", "zw", +].into_iter().collect::>()); -pub fn normalize_channel_name(name: &str, terms_to_remove: &[&str]) -> String { +fn split_country_prefix(s: &str) -> (Option, &str) { + if s.len() < 5 || !s[2..3].chars().any(|c| !c.is_alphabetic()) { + return (None, s); + } + let first_code = &s[0..2]; + if !COUNTRY_CODE.contains(first_code) { + return (None, s); + } + (Some(first_code.to_string()), &s[3..]) +} + +fn country_prefix(name: String, normalize_config: &EpgNormalizeConfig) -> (String, Option) { + if normalize_config.country_prefix != EpgCountryPrefix::Ignore { + let (prefix, suffix) = split_country_prefix(&name); + if prefix.is_some() { + return (suffix.to_string(), prefix); + } + } + (name, None) +} + +fn combine(join: &str, left: &str, right: &str) -> String { + let mut combined = String::with_capacity(left.len() + join.len() + right.len()); + combined.push_str(left); + combined.push('.'); + combined.push_str(right); + combined +} + +/// # Panics +pub fn normalize_channel_name(name: &str, normalize_config: &EpgNormalizeConfig) -> String { let normalized = deunicode(name).to_lowercase(); - + let (channel_name, suffix) = country_prefix(normalized, normalize_config); // Remove all non-alphanumeric characters (except dashes and underscores). - let cleaned_name = NORMALIZE_CHANNEL.replace_all(&normalized, ""); - + let cleaned_name = normalize_config.t_normalize_regex.as_ref().unwrap().replace_all(&channel_name, ""); // Remove terms like resolution - let result = terms_to_remove.iter().fold(cleaned_name.to_string(), |acc, term| { - acc.replace(*term, "") + let cleaned_name = normalize_config.t_strip.iter().fold(cleaned_name.to_string(), |acc, term| { + acc.replace(term, "") }); - - result + match suffix { + None => cleaned_name, + Some(sfx) => { + match &normalize_config.country_prefix { + EpgCountryPrefix::Ignore => cleaned_name, + EpgCountryPrefix::Suffix(sep) => combine(sep, &cleaned_name, &sfx), + EpgCountryPrefix::Prefix(sep) => combine(sep, &sfx, &cleaned_name), + } + } + } } impl TVGuide { @@ -40,29 +92,76 @@ impl TVGuide { } fn process_epg_file(id_cache: &mut EpgIdCache, epg_file: &Path) -> Option { - let strip = id_cache.strip.clone(); match CompressedFileReader::new(epg_file) { Ok(mut reader) => { let mut children: Vec = vec![]; let mut tv_attributes: Option>> = None; + let metaphone = Metaphone::default(); + let normalization = id_cache.normalize_config.enabled; + let fuzzy_matching = id_cache.normalize_config.fuzzy_matching; let mut filter_tags = |tag: XmlTag| { match tag.name.as_str() { EPG_TAG_CHANNEL => { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { if !id_cache.processed.contains(epg_id) { - let id: Cow = Cow::Owned(epg_id.to_string()); - for normalized_epg_id in &tag.normalized_epg_ids { - let key = Cow::Owned(normalized_epg_id.to_string()); - match id_cache.normalized.entry(key) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - entry.insert(Some(id.clone())); - id_cache.channel.insert(id.clone()); + if normalization { + let mut matched = false; + let id: Cow = Cow::Owned(epg_id.to_string()); + for normalized_epg_id in &tag.normalized_epg_ids { + let key = Cow::Owned(normalized_epg_id.to_string()); + match id_cache.normalized.entry(key) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().1 = Some(id.clone()); + id_cache.channel.insert(id.clone()); + matched = true; + break; + } + std::collections::hash_map::Entry::Vacant(_entry) => {} } - std::collections::hash_map::Entry::Vacant(_entry) => {} } - } - if id_cache.channel.contains(&id) { - children.push(tag); + + if !matched && fuzzy_matching { + let mut matched_normalized_epg_id: Option> = None; + let mut threshold = 0.0; + + 'outer: for (norm_key, (phonetic_code, _)) in &id_cache.normalized { + for normalized_epg_id in &tag.normalized_epg_ids { + let code = metaphone.encode(normalized_epg_id); + if &code == phonetic_code { + let match_jw = strsim::jaro_winkler(norm_key, normalized_epg_id); + if match_jw >= id_cache.normalize_config.t_match_threshold { + threshold = if threshold > match_jw { threshold } else { + matched_normalized_epg_id = Some(Cow::Borrowed(norm_key)); + match_jw + }; + if threshold > 99.9 { + break 'outer; + } + } + } + } + // is there an early exit strategy ??? + } + if matched { + match id_cache.normalized.entry(Cow::Owned(matched_normalized_epg_id.unwrap().to_string())) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().1 = Some(id.clone()); + id_cache.channel.insert(id.clone()); + matched = true; + } + std::collections::hash_map::Entry::Vacant(_entry) => {} + } + } + } + + if matched { + children.push(tag); + } + } else { + let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); + if id_cache.channel.contains(&borrowed_epg_id) { + children.push(tag); + } } } } @@ -84,7 +183,7 @@ impl TVGuide { } }; - parse_tvguide(&mut reader, &mut filter_tags, &strip); + parse_tvguide(&mut reader, &mut filter_tags, &id_cache.normalize_config); if children.is_empty() { return None; @@ -120,7 +219,7 @@ impl TVGuide { } } -pub fn parse_tvguide(content: R, callback: &mut F, strip: &[&str]) +pub fn parse_tvguide(content: R, callback: &mut F, epg_normalize_config: &EpgNormalizeConfig) where R: std::io::BufRead, F: FnMut(XmlTag), @@ -128,6 +227,7 @@ where let mut stack: Vec = vec![]; let mut reader = Reader::from_reader(content); let mut buf = Vec::::new(); + let normalize_enabled = epg_normalize_config.enabled; loop { match reader.read_event_into(&mut buf) { Ok(Event::Eof) => break, @@ -159,8 +259,10 @@ where for child in children { match child.name.as_str() { EPG_TAG_DISPLAY_NAME => { - if let Some(name) = &child.value { - tag.normalized_epg_ids.insert(normalize_channel_name(name, strip)); + if normalize_enabled { + if let Some(name) = &child.value { + tag.normalized_epg_ids.insert(normalize_channel_name(name, epg_normalize_config)); + } } } EPG_TAG_ICON => { @@ -283,6 +385,7 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option { #[cfg(test)] mod tests { + use crate::model::config::{EpgCountryPrefix, EpgNormalizeConfig}; use crate::processing::parser::xmltv::normalize_channel_name; // #[test] @@ -306,9 +409,36 @@ mod tests { #[test] fn normalize() { - assert_eq!("satsupersport6", normalize_channel_name("SAT: SUPERSPORT 6 ᴿᴬᵂ")); - assert_eq!("odisea", normalize_channel_name("4K: ODISEA ᴿᴬᵂ")); - assert_eq!("odisea", normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ")); + let mut epg_normalize_cfg = EpgNormalizeConfig::default(); + epg_normalize_cfg.country_prefix = EpgCountryPrefix::Suffix(".".to_string()); + println!("{:?}", epg_normalize_cfg); + assert_eq!("supersport6.ru", normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_normalize_cfg)); + assert_eq!("satodisea", normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_normalize_cfg)); + assert_eq!("odisea", normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg)); + assert_eq!("odisea", normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg)); + assert_eq!("buodisea", normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg)); + assert_eq!("odisea.bg", normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg)); + } + use rphonetic::{Encoder, Metaphone}; + #[test] + fn test_metaphone() { + let metaphone = Metaphone::default(); + let mut epg_normalize_cfg = EpgNormalizeConfig::default(); + epg_normalize_cfg.country_prefix = EpgCountryPrefix::Suffix(".".to_string()); + println!("{:?}", epg_normalize_cfg); + // assert_eq!("supersport6.ru", metaphone.encode(&normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_normalize_cfg))); + // assert_eq!("satodisea", metaphone.encode(&normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_normalize_cfg))); + // assert_eq!("odisea", metaphone.encode(&normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + // assert_eq!("odisea", metaphone.encode(&normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + // assert_eq!("buodisea", metaphone.encode(&normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + // assert_eq!("odisea.bg", metaphone.encode(&normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + + println!("{}", metaphone.encode(&normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_normalize_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_normalize_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); } } \ No newline at end of file diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index 51aab48a3..1c41e4afc 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -1,5 +1,5 @@ use crate::Config; -use crate::model::config::{ConfigInput, ConfigRename}; +use crate::model::config::{ConfigInput, ConfigRename, EpgConfig, EpgNormalizeConfig}; use crate::utils::network::epg; use crate::utils::network::m3u; use crate::utils::network::xtream; @@ -14,6 +14,7 @@ use std::thread; use log::{debug, error, info, log_enabled, trace, warn, Level}; use std::time::Instant; use deunicode::deunicode; +use rphonetic::{Encoder, Metaphone}; use crate::foundation::filter::{get_field_value, set_field_value, MockValueProcessor, ValueProvider}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, get_errors_notify_message, notify_err}; use crate::messaging::{send_message, MsgKind}; @@ -502,30 +503,26 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { pub struct EpgIdCache<'a > { pub channel: HashSet>, - pub normalized: HashMap, Option>>, + pub normalized: HashMap, (String, Option>)>, + pub normalized_phonetic: HashMap, String>, pub processed: HashSet, - pub strip: Vec<&'a str>, + pub normalize_config: EpgNormalizeConfig, } - -const TERMS_TO_REMOVE: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]; - -impl<'a> EpgIdCache<'a> { - pub fn new(strip: Option<&'a Vec>) -> Self { - let str_refs: Vec<&'a str> = match strip { - Some(vec) => vec.iter().map(std::string::String::as_str).collect(), - None => TERMS_TO_REMOVE.to_vec(), - }; +impl EpgIdCache<'_> { + pub fn new(epg_config: Option<&EpgConfig>) -> Self { + let normalize_config = epg_config.map_or_else(EpgNormalizeConfig::default, |epg_config| epg_config.t_normalize.clone()); EpgIdCache { channel: HashSet::new(), normalized: HashMap::new(), + normalized_phonetic: HashMap::new(), processed: HashSet::new(), - strip: str_refs, + normalize_config, } } pub fn normalize(&self, name: &str) -> String { - normalize_channel_name(name, &self.strip) + normalize_channel_name(name, &self.normalize_config) } } @@ -560,23 +557,40 @@ async fn process_playlist_for_target(client: Arc, let mut new_playlist = vec![]; let mut new_epg = vec![]; + let metaphone = Metaphone::default(); + // each fetched playlist can have its own epgl url. // we need to process each input epg. for mut fp in processed_fetched_playlists { // collect all epg_channel ids - let mut id_cache = EpgIdCache::new(fp.input.epg_strip.as_ref()); + let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); + let normalize_enabled = id_cache.normalize_config.enabled; + let fuzzy_matching = id_cache.normalize_config.fuzzy_matching; for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { - match channel.header.epg_channel_id.as_ref() { - None => { - id_cache.normalized.insert(Cow::Owned(id_cache.normalize(&channel.header.name)), None); - }, - Some(epg_id) => { - if epg_id.is_empty() { - id_cache.normalized.insert(Cow::Owned(id_cache.normalize(&channel.header.name)), None); - } else { - id_cache.channel.insert(Cow::Owned(epg_id.to_string())); - } - }, + let epg_id = channel.header.epg_channel_id.as_deref(); + let name = &channel.header.name; + + if let Some(id) = epg_id { + if !id.is_empty() { + id_cache.channel.insert(Cow::Owned(id.to_string())); + } + } + + // hen fuzzy_matching we need to put the normalized name even if there is an epg_id, because the epg_id + // could not match to the epg file. And then we try to guess it based on normalized name + let needs_normalization = normalize_enabled && (fuzzy_matching || epg_id.is_none_or(str::is_empty)); + + if needs_normalization { + let normalized_name = id_cache.normalize(name); + let normalized_key = Cow::Owned(normalized_name.clone()); + + let phonetic_code = if fuzzy_matching { + metaphone.encode(&normalized_name) + } else { + String::new() + }; + + id_cache.normalized.insert(normalized_key, (phonetic_code, None)); } } // let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) @@ -594,9 +608,9 @@ async fn process_playlist_for_target(client: Arc, .filter(|c| c.header.xtream_cluster == XtreamCluster::Live ) .filter(|c| c.header.epg_channel_id.is_none() || c.header.logo.is_empty() || c.header.logo_small.is_empty()) .for_each(|c| { - if c.header.epg_channel_id.as_ref().is_none() { + if c.header.epg_channel_id.as_ref().is_none() || !id_cache.processed.contains(c.header.epg_channel_id.as_ref().unwrap()) { let normalized = id_cache.normalize(&c.header.name); - if let Some(Some(epg_id)) = id_cache.normalized.get(&Cow::Borrowed(normalized.as_str())) { + if let Some((_, Some(epg_id))) = id_cache.normalized.get(&Cow::Borrowed(normalized.as_str())) { c.header.epg_channel_id = Some(epg_id.to_string()); } } @@ -667,3 +681,19 @@ pub async fn exec_processing(client: Arc, cfg: Arc, tar let elapsed = start_time.elapsed().as_secs(); info!("Update process finished! Took {elapsed} secs."); } + +#[cfg(test)] +mod tests { + #[test] + fn test() { + let data = [("yessport5", "heyessport5gold"), ("yessport5", "heyesport5gold")]; + + data.iter().for_each(|(first, second)| + println!("jaro_winkler {} = {} => {}", first, second, strsim::jaro_winkler(first, second))); + // println!("jaro {}", strsim::jaro(data.0, data.1)); + // println!("levenhstein {}", strsim::levenshtein(data.0, data.1)); + // println!("damerau_levenshtein {:?}", strsim::damerau_levenshtein(data.0, data.1)); + // println!("osa distance {:?}", strsim::osa_distance(data.0, data.1)); + // println!("sorensen dice {:?}", strsim::sorensen_dice(data.0, data.1)); + } +} \ No newline at end of file diff --git a/src/utils/default_utils.rs b/src/utils/default_utils.rs index e2dc4ec7f..39f07842f 100644 --- a/src/utils/default_utils.rs +++ b/src/utils/default_utils.rs @@ -4,4 +4,4 @@ pub fn default_as_default() -> String { String::from("default") } pub const fn default_as_two_u16() -> u16 { 2 } -pub const fn default_grace_period_millis() -> u64 { 0 } +pub const fn default_grace_period_millis() -> u64 { 0 } \ No newline at end of file diff --git a/src/utils/network/epg.rs b/src/utils/network/epg.rs index 67f38ceb9..d4d784b04 100644 --- a/src/utils/network/epg.rs +++ b/src/utils/network/epg.rs @@ -1,13 +1,13 @@ -use std::path::PathBuf; -use std::sync::Arc; -use log::debug; use crate::m3u_filter_error::M3uFilterError; -use crate::model::config::{Config, ConfigInput, EpgUrl}; +use crate::model::config::{Config, ConfigInput}; use crate::model::xmltv::TVGuide; -use crate::repository::storage::{short_hash}; +use crate::repository::storage::short_hash; +use crate::utils::file::file_utils; use crate::utils::file::file_utils::prepare_file_path; use crate::utils::network::request; -use crate::utils::file::file_utils; +use log::debug; +use std::path::PathBuf; +use std::sync::Arc; async fn download_epg_file(url: &str, client: &Arc, input: &ConfigInput, working_dir: &str) -> Result { @@ -20,17 +20,14 @@ async fn download_epg_file(url: &str, client: &Arc, input: &Con } pub async fn get_xmltv(client: Arc, _cfg: &Config, input: &ConfigInput, working_dir: &str) -> (Option, Vec) { - match &input.epg_url { + match &input.epg { None => (None, vec![]), - Some(urls) => { + Some(epg_config) => { let mut errors = vec![]; let mut file_paths = vec![]; - for url in match urls { - EpgUrl::Single(value) => vec![value.to_string()], - EpgUrl::Multi(value) => value.clone(), - } { - match download_epg_file(&url, &client, input, working_dir).await { + for url in &epg_config.t_urls { + match download_epg_file(url, &client, input, working_dir).await { Ok(file_path) => { file_paths.push(file_path); }