diff --git a/README.md b/README.md index 6b8514f2c..246cb97bb 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ nobody: $argon2id$v=$argon2id$v=19$m=19456,t=2,p=1$Y2FROE83ZDQ1c2VaYmJ4VU9YdHpuZ The password can be generated with ```shell -./m3u-filter -p /op/m3u-filter/config --genpwd` +./m3u-filter --genpwd` ``` or with docker diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index 6b31fbfe5..1731b0680 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -12,7 +12,23 @@ use std::collections::{HashMap, HashSet}; use std::mem; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, LazyLock, Mutex, RwLock}; + +static COUNTRY_CODE: LazyLock> = LazyLock::new(|| vec![ + "af", "al", "dz", "ad", "ao", "ag", "ar", "am", "au", "at", "az", "bs", "bh", "bd", "bb", "by", + "be", "bz", "bj", "bt", "bo", "ba", "bw", "br", "bn", "bg", "bf", "bi", "cv", "kh", "cm", "ca", + "cf", "td", "cl", "cn", "co", "km", "cg", "cr", "hr", "cu", "cy", "cz", "cd", "dk", "dj", "dm", + "do", "tl", "ec", "eg", "sv", "gq", "er", "ee", "sz", "et", "fj", "fi", "fr", "ga", "gm", "ge", + "de", "gh", "gr", "gd", "gt", "gn", "gw", "gy", "ht", "hn", "hu", "is", "in", "id", "ir", "iq", + "ie", "il", "it", "ci", "jm", "jp", "jo", "kz", "ke", "ki", "kp", "kr", "kw", "kg", "la", "lv", + "lb", "ls", "lr", "ly", "li", "lt", "lu", "mg", "mw", "my", "mv", "ml", "mt", "mh", "mr", "mu", + "mx", "fm", "md", "mc", "mn", "me", "ma", "mz", "mm", "na", "nr", "np", "nl", "nz", "ni", "ne", + "ng", "mk", "no", "om", "pk", "pw", "pa", "pg", "py", "pe", "ph", "pl", "pt", "qa", "ro", "ru", + "rw", "kn", "lc", "vc", "ws", "sm", "st", "sa", "sn", "rs", "sc", "sl", "sg", "sk", "si", "sb", + "so", "za", "ss", "es", "lk", "sd", "sr", "se", "ch", "sy", "tw", "tj", "tz", "th", "tg", "to", + "tt", "tn", "tr", "tm", "tv", "ug", "ua", "ae", "gb", "us", "uy", "uz", "vu", "va", "ve", "vn", + "ye", "zm", "zw", +].into_iter().collect::>()); fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a str>, &'a str) { for delim in delimiters { @@ -20,7 +36,11 @@ fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a let (left, right) = input.split_at(index); let right = &right[delim.len_utf8()..].trim(); if !right.is_empty() { - return (Some(left.trim()), right.trim()); + let prefix = left.trim(); + // when we used anything as prefix the result was bad + if COUNTRY_CODE.contains(&prefix) { + return (Some(prefix), right.trim()); + } } } } diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index 538f25dad..da8b5187d 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -518,6 +518,8 @@ pub struct EpgIdCache<'a > { pub processed: HashSet, pub smart_match_config: EpgSmartMatchConfig, pub metaphone: Metaphone, + pub smart_match_enabled: bool, + pub fuzzy_match_enabled: bool, } impl EpgIdCache<'_> { @@ -528,11 +530,18 @@ impl EpgIdCache<'_> { normalized: HashMap::new(), normalized_phonetic: HashMap::new(), processed: HashSet::new(), - smart_match_config: normalize_config, metaphone: Metaphone::default(), + smart_match_enabled: normalize_config.enabled, + fuzzy_match_enabled: normalize_config.enabled && normalize_config.fuzzy_matching, + smart_match_config: normalize_config, + } } + pub fn is_empty(&self) -> bool { + self.channel.is_empty() && self.normalized.is_empty() + } + pub fn normalize_and_store(&mut self, name: &str) { let normalized_name = self.normalize(name); let normalized_key: Cow = Cow::Owned(normalized_name.clone()); @@ -568,10 +577,12 @@ impl EpgIdCache<'_> { } } -fn prepare_epg_id_cache(fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache, normalize_enabled: bool, fuzzy_matching: bool) { +fn prepare_epg_id_cache(fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache) { + let smart_match_enabled = id_cache.smart_match_enabled; + let fuzzy_matching = id_cache.fuzzy_match_enabled; + for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { let epg_id = channel.header.epg_channel_id.as_deref(); - let name = &channel.header.name; // insert epg_id to known channel epg_ids if let Some(id) = epg_id { @@ -582,47 +593,56 @@ fn prepare_epg_id_cache(fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache, nor // for fuzzy_matching we need to put the normalized name even if there is an epg_id, because the epg_id // could not match to the epg file. And then we try to guess it based on normalized name - let needs_normalization = normalize_enabled && (fuzzy_matching || epg_id.is_none_or(str::is_empty)); + let needs_normalization = smart_match_enabled && (fuzzy_matching || epg_id.is_none_or(str::is_empty)); if needs_normalization { + let name = &channel.header.name; id_cache.normalize_and_store(name); } } } -fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist, smart_match_enabled: bool, id_cache: &mut EpgIdCache) { +fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache) { + if let Some(tv_guide) = &fp.epg { if let Some(epg) = tv_guide.filter(id_cache) { + // icon tags let icon_tags : HashMap<&String, &XmlTag> = epg.children.iter() .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t)).collect(); + + let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live; + let filter_missing_epg_id = |c: &&mut PlaylistItem | c.header.epg_channel_id.is_none() || c.header.logo.is_empty() || c.header.logo_small.is_empty(); + + let assign_values = |c: &mut PlaylistItem| { + if id_cache.smart_match_enabled { + // if the channel has no epg_id or the epg_id is not present in xmltv/tvguide then we need to match one from existing tvguide + if c.header.epg_channel_id.is_none() || !id_cache.processed.contains(c.header.epg_channel_id.as_ref().unwrap()) { + let normalized = id_cache.normalize(&c.header.name); + if let Some((_, Some(epg_id))) = id_cache.normalized.get(&Cow::Borrowed(normalized.as_str())) { + c.header.epg_channel_id = Some(epg_id.to_string()); + } + } + } + if c.header.epg_channel_id.is_some() && (c.header.logo.is_empty() || c.header.logo_small.is_empty()) { + if let Some(icon_tag) = icon_tags.get(c.header.epg_channel_id.as_ref().unwrap()) { + if let Some(icon) = icon_tag.icon.as_ref() { + if c.header.logo.is_empty() { + c.header.logo = (*icon).to_string(); + } + if c.header.logo_small.is_empty() { + c.header.logo = (*icon).to_string(); + } + } + } + } + }; + fp.playlistgroups.iter_mut() .flat_map(|g| &mut g.channels) - .filter(|c| c.header.xtream_cluster == XtreamCluster::Live) - .filter(|c| c.header.epg_channel_id.is_none() || c.header.logo.is_empty() || c.header.logo_small.is_empty()) - .for_each(|c| { - if smart_match_enabled { - // if the channel has no epg_id or the epg_id is not present in xmltv/tvguide then we need to match one from existing tvguide - if c.header.epg_channel_id.is_none() || !id_cache.processed.contains(c.header.epg_channel_id.as_ref().unwrap()) { - let normalized = id_cache.normalize(&c.header.name); - if let Some((_, Some(epg_id))) = id_cache.normalized.get(&Cow::Borrowed(normalized.as_str())) { - c.header.epg_channel_id = Some(epg_id.to_string()); - } - } - } - if c.header.epg_channel_id.is_some() && (c.header.logo.is_empty() || c.header.logo_small.is_empty()) { - if let Some(icon_tag) = icon_tags.get(c.header.epg_channel_id.as_ref().unwrap()) { - if let Some(icon) = icon_tag.icon.as_ref() { - if c.header.logo.is_empty() { - c.header.logo = (*icon).to_string(); - } - if c.header.logo_small.is_empty() { - c.header.logo = (*icon).to_string(); - } - } - } - } - }); + .filter(filter_live) + .filter(filter_missing_epg_id) + .for_each(assign_values); new_epg.push(epg); } } @@ -656,27 +676,7 @@ async fn process_playlist_for_target(client: Arc, apply_affixes(&mut processed_fetched_playlists); - let mut new_playlist = vec![]; - let mut new_epg = vec![]; - - // each fetched playlist can have its own epgl url. - // we need to process each input epg. - for mut fp in processed_fetched_playlists { - // collect all epg_channel ids - let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); - let smart_match_enabled = id_cache.smart_match_config.enabled; - let fuzzy_matching = smart_match_enabled && id_cache.smart_match_config.fuzzy_matching; - prepare_epg_id_cache(&mut fp, &mut id_cache, smart_match_enabled, fuzzy_matching); - // let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) - // .filter_map(|c| c.header.epg_channel_id.as_ref()).map(|a| a.as_str()).collect(); - if id_cache.channel.is_empty() && id_cache.normalized.is_empty() { - debug!("channel ids are empty"); - } else { - debug_if_enabled!("found epg information for {}", &target.name); - assign_channel_epg(&mut new_epg, &mut fp, smart_match_enabled, &mut id_cache); - } - new_playlist.append(&mut fp.playlistgroups); - } + let (new_epg, new_playlist) = process_epg(target, &mut processed_fetched_playlists); if new_playlist.is_empty() { info!("Playlist is empty: {}", &target.name); @@ -691,6 +691,28 @@ async fn process_playlist_for_target(client: Arc, } } +fn process_epg(target: &ConfigTarget, processed_fetched_playlists: &mut Vec) -> (Vec, Vec) { + let mut new_playlist = vec![]; + let mut new_epg = vec![]; + + // each fetched playlist can have its own epgl url. + // we need to process each input epg. + for fp in processed_fetched_playlists { + // collect all epg_channel ids + let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); + prepare_epg_id_cache(fp, &mut id_cache); + + if id_cache.is_empty() { + debug!("channel ids are empty"); + } else { + debug_if_enabled!("found epg information for {}", &target.name); + assign_channel_epg(&mut new_epg, fp, &mut id_cache); + } + new_playlist.append(&mut fp.playlistgroups); + } + (new_epg, new_playlist) +} + fn process_watch(target: &ConfigTarget, cfg: &Config, new_playlist: &Vec) { if target.t_watch_re.is_some() { if default_as_default().eq_ignore_ascii_case(&target.name) { diff --git a/src/utils/string_utils.rs b/src/utils/string_utils.rs index 2f8d77ebb..f6129faac 100644 --- a/src/utils/string_utils.rs +++ b/src/utils/string_utils.rs @@ -44,7 +44,13 @@ pub fn get_trimmed_string(value: &Option) -> Option { pub fn replace_after_last_slash(input: &str, replacement: &str) -> String { match input.rsplitn(2, '/').collect::>().as_slice() { - [_after, before] => format!("{before}/{replacement}"), + [_after, before] => { + if replacement.starts_with("/") { + format!("{before}{replacement}") + } else { + format!("{before}/{replacement}") + } + }, [_only] => replacement.to_string(), // if there is no slash, replace complete _ => input.to_string(), // fallback, should never happen }