diff --git a/CHANGELOG.md b/CHANGELOG.md index 59a27135c..370d3a755 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,14 +30,17 @@ web_ui: - !BREAKING_CHANGE! user has now the attribute `ui_enabled` to disable/enable web_ui for user. You need to migrate the user db if you have used `use_user_db:true`. Set it to `false` run old m3u-filter version, then update m3u-filter and set `use_user_db:true`and start. -- !BREAKING_CHANGE! multi epg processing/optimization, auto guessing/assigning epg id's +- multi epg processing/optimization, auto guessing/assigning epg id's ```yaml -epg_url: ['https://localhost.com/epg.xml'] +# single epg +epg_url: 'https://localhost.com/epg.xml' ``` ```yaml +# multi local file epg epg_url: ['file:///${env:M3U_FILTER_HOME}/epg.xml', 'file:///${env:M3U_FILTER_HOME}/epg2.xml'] ``` ```yaml +# multi url epg epg_url: ['http://localhost:3001/xmltv.php?epg_id=1', 'http://localhost:3001/xmltv.php?epg_id=2'] ``` diff --git a/Cargo.lock b/Cargo.lock index d84836e3e..37a2d0dcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -639,6 +639,12 @@ dependencies = [ "syn", ] +[[package]] +name = "deunicode" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc55fe0d1f6c107595572ec8b107c0999bb1a2e0b75e37429a4fb0d6474a0e7d" + [[package]] name = "digest" version = "0.10.7" @@ -1532,6 +1538,7 @@ dependencies = [ "chrono", "clap", "cron", + "deunicode", "enum-iterator", "env_logger", "filetime", @@ -1564,7 +1571,6 @@ dependencies = [ "tower", "tower-http", "tower_governor", - "unidecode", "url", "vergen", "winapi", @@ -2914,12 +2920,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unidecode" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402bb19d8e03f1d1a7450e2bd613980869438e0666331be3e073089124aa1adc" - [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 5392be6ff..7ab0c59bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ pest = "2.7" pest_derive = "2.7" enum-iterator = "2" openssl = { version = "*", features = ["vendored"] } #https://docs.rs/openssl/0.10.34/openssl/#vendored -unidecode = "0" +deunicode = "1.6.1" mime = "0.3" log = "0.4" env_logger = "0.11" diff --git a/src/api/model/download.rs b/src/api/model/download.rs index ed9700263..1ce54dc14 100644 --- a/src/api/model/download.rs +++ b/src/api/model/download.rs @@ -3,8 +3,8 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; use tokio::sync::{RwLock, Mutex}; use std::sync::Arc; +use deunicode::deunicode; use serde::{Deserialize, Serialize}; -use unidecode::unidecode; use crate::model::config::VideoDownloadConfig; use crate::repository::storage::hash_string_as_hex; @@ -74,7 +74,7 @@ impl FileDownload { match reqwest::Url::parse(req_url) { Ok(url) => { let filename_re = download_cfg.t_re_filename.as_ref().unwrap(); - let tmp_filename = filename_re.replace_all(&unidecode(req_filename) + let tmp_filename = filename_re.replace_all(&deunicode(req_filename) .replace(' ', "_"), "") .replace("__", "_") .replace("_-_", "-"); diff --git a/src/model/config.rs b/src/model/config.rs index 4f8d34c20..d443f6757 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -756,6 +756,13 @@ impl ConfigInputAlias { } } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum EpgUrl { + Single(String), + Multi(Vec) +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct ConfigInput { @@ -768,7 +775,7 @@ pub struct ConfigInput { pub headers: HashMap, pub url: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub epg_url: Option>, + pub epg_url: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub username: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -808,11 +815,19 @@ impl ConfigInput { check_input_credentials!(self, self.input_type); self.persist = get_trimmed_string(&self.persist); - self.epg_url = self.epg_url.take().map(|list| { - list.into_iter() - .map(|url| url.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect() + self.epg_url = self.epg_url.take().map(|epg_url| { + match epg_url { + EpgUrl::Single(url) => if url.trim().is_empty() { + EpgUrl::Multi(vec![]) + } else { + EpgUrl::Single(url.trim().to_string()) + }, + EpgUrl::Multi(urls) => + EpgUrl::Multi(urls.into_iter() + .map(|url| url.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect()) + } }); if let Some(aliases) = self.aliases.as_mut() { diff --git a/src/model/xmltv.rs b/src/model/xmltv.rs index ea1ab869b..0035d42b3 100644 --- a/src/model/xmltv.rs +++ b/src/model/xmltv.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap}; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; use quick_xml::{Error, Writer}; @@ -21,10 +21,11 @@ pub struct XmlTag { pub attributes: Option>>, pub children: Option>>, pub icon: Option, - pub normalized_epg_id: Option, + pub normalized_epg_ids: HashSet, } impl XmlTag { + pub fn get_attribute_value(&self, attr_name: &str) -> Option<&String> { self.attributes.as_ref().and_then(|attr| attr.get(attr_name)) } diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index fc3c63ea5..075dbbe8f 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -7,14 +7,15 @@ use regex::Regex; use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, LazyLock}; -use unidecode::unidecode; +use deunicode::deunicode; static NORMALIZE_CHANNEL: LazyLock = LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9\-]").unwrap()); +// TODO into config const TERMS_TO_REMOVE: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]; pub fn normalize_channel_name(name: &str) -> String { - let normalized = unidecode(name).to_lowercase(); + let normalized = deunicode(name).to_lowercase(); // Remove all non-alphanumeric characters (except dashes and underscores). let cleaned_name = NORMALIZE_CHANNEL.replace_all(&normalized, ""); @@ -40,7 +41,8 @@ impl TVGuide { }) } - fn process_epg_file(epg_channel_ids: &mut HashSet>, normalized_epg_channel_ids: &mut HashMap, Option>>, processed_epg_channel_ids: &mut HashSet, epg_file: &Path) -> Option { + fn process_epg_file(epg_channel_ids: &mut HashSet>, normalized_epg_channel_ids: &mut HashMap, Option>>, + processed_epg_channel_ids: &mut HashSet, epg_file: &Path) -> Option { match CompressedFileReader::new(epg_file) { Ok(mut reader) => { let mut children: Vec = vec![]; @@ -51,7 +53,7 @@ impl TVGuide { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { if !processed_epg_channel_ids.contains(epg_id) { let id: Cow = Cow::Owned(epg_id.to_string()); - if let Some(normalized_epg_id) = tag.normalized_epg_id.as_ref() { + for normalized_epg_id in &tag.normalized_epg_ids { let key = Cow::Owned(normalized_epg_id.to_string()); match normalized_epg_channel_ids.entry(key) { std::collections::hash_map::Entry::Occupied(mut entry) => { @@ -111,8 +113,7 @@ impl TVGuide { } let mut processed_epg_ids: HashSet = HashSet::new(); let epgs: Vec = self.file_paths.iter() - .map(|path| Self::process_epg_file(epg_channel_ids, normalized_epg_channel_ids, &mut processed_epg_ids, path)) - .flatten() + .filter_map(|path| Self::process_epg_file(epg_channel_ids, normalized_epg_channel_ids, &mut processed_epg_ids, path)) .collect(); if epgs.len() == 1 { epgs.into_iter().next() @@ -144,7 +145,7 @@ where attributes: attribs, children: None, icon: None, - normalized_epg_id: None, + normalized_epg_ids: HashSet::new(), }; if is_tv_tag { @@ -162,7 +163,7 @@ where match child.name.as_str() { EPG_TAG_DISPLAY_NAME => { if let Some(name) = &child.value { - tag.normalized_epg_id = Some(normalize_channel_name(name)); + tag.normalized_epg_ids.insert(normalize_channel_name(name)); } } EPG_TAG_ICON => { @@ -309,5 +310,8 @@ mod tests { #[test] fn normalize() { assert_eq!("satsupersport6", normalize_channel_name("SAT: SUPERSPORT 6 ᴿᴬᵂ")); + assert_eq!("odisea", normalize_channel_name("4K: ODISEA ᴿᴬᵂ")); + assert_eq!("odisea", normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ")); + } } \ No newline at end of file diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index e807fb427..7c3a718cb 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -1,5 +1,3 @@ -extern crate unidecode; - use crate::Config; use crate::model::config::{ConfigInput, ConfigRename}; use crate::utils::network::epg; @@ -15,8 +13,7 @@ use std::thread; use log::{debug, error, info, log_enabled, trace, warn, Level}; use std::time::Instant; -use unidecode::unidecode; - +use deunicode::deunicode; use crate::foundation::filter::{get_field_value, set_field_value, MockValueProcessor, ValueProvider}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, get_errors_notify_message, notify_err}; use crate::messaging::{send_message, MsgKind}; @@ -62,8 +59,8 @@ fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt } fn playlistgroup_comparator(a: &PlaylistGroup, b: &PlaylistGroup, group_sort: &ConfigSortGroup, match_as_ascii: bool) -> Ordering { - let value_a = if match_as_ascii { unidecode(&a.title) } else { a.title.to_string() }; - let value_b = if match_as_ascii { unidecode(&b.title) } else { b.title.to_string() }; + let value_a = if match_as_ascii { deunicode(&a.title) } else { a.title.to_string() }; + let value_b = if match_as_ascii { deunicode(&b.title) } else { b.title.to_string() }; let ordering = value_a.partial_cmp(&value_b).unwrap(); match group_sort.order { Asc => ordering, @@ -74,8 +71,8 @@ fn playlistgroup_comparator(a: &PlaylistGroup, b: &PlaylistGroup, group_sort: &C fn playlistitem_comparator(a: &PlaylistItem, b: &PlaylistItem, channel_sort: &ConfigSortChannel, match_as_ascii: bool) -> Ordering { let raw_value_a = get_field_value(a, &channel_sort.field); let raw_value_b = get_field_value(b, &channel_sort.field); - let value_a = if match_as_ascii { unidecode(&raw_value_a) } else { raw_value_a }; - let value_b = if match_as_ascii { unidecode(&raw_value_b) } else { raw_value_b }; + let value_a = if match_as_ascii { deunicode(&raw_value_a) } else { raw_value_a }; + let value_b = if match_as_ascii { deunicode(&raw_value_b) } else { raw_value_b }; channel_sort.sequence.as_ref().map_or_else(|| { let ordering = value_a.partial_cmp(&value_b).unwrap(); match channel_sort.order { @@ -122,7 +119,7 @@ fn sort_playlist(target: &ConfigTarget, new_playlist: &mut [PlaylistGroup]) { for channel_sort in channel_sorts { let regexp = channel_sort.re.as_ref().unwrap(); for group in new_playlist.iter_mut() { - let group_title = if match_as_ascii { unidecode(&group.title) } else { group.title.to_string() }; + let group_title = if match_as_ascii { deunicode(&group.title) } else { group.title.to_string() }; if regexp.is_match(group_title.as_str()) { group.channels.sort_by(|chan1, chan2| playlistitem_comparator(chan1, chan2, channel_sort, match_as_ascii)); } @@ -196,7 +193,7 @@ macro_rules! apply_pattern { fn map_channel(mut channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem { if !mapping.mapper.is_empty() { let header = &channel.header; - let channel_name = if mapping.match_as_ascii { unidecode(&header.name) } else { header.name.to_string() }; + let channel_name = if mapping.match_as_ascii { deunicode(&header.name) } else { header.name.to_string() }; if mapping.match_as_ascii && log_enabled!(Level::Trace) { trace!("Decoded {} for matching to {}", &header.name, &channel_name); } // let ref_chan = &mut channel; let ref_chan = &mut channel; @@ -542,8 +539,16 @@ async fn process_playlist_for_target(client: Arc, let mut normalized_epg_channel_ids: HashMap, Option>> = HashMap::new(); for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { match channel.header.epg_channel_id.as_ref() { - None => {normalized_epg_channel_ids.insert(Cow::Owned(normalize_channel_name(&channel.header.name)), None);}, - Some(epg_id) => {epg_channel_ids.insert(Cow::Owned(epg_id.to_string()));}, + None => { + normalized_epg_channel_ids.insert(Cow::Owned(normalize_channel_name(&channel.header.name)), None); + }, + Some(epg_id) => { + if epg_id.is_empty() { + normalized_epg_channel_ids.insert(Cow::Owned(normalize_channel_name(&channel.header.name)), None); + } else { + epg_channel_ids.insert(Cow::Owned(epg_id.to_string())); + } + }, } } // let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) diff --git a/src/utils/network/epg.rs b/src/utils/network/epg.rs index bb9d33d71..67f38ceb9 100644 --- a/src/utils/network/epg.rs +++ b/src/utils/network/epg.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::sync::Arc; use log::debug; use crate::m3u_filter_error::M3uFilterError; -use crate::model::config::{Config, ConfigInput}; +use crate::model::config::{Config, ConfigInput, EpgUrl}; use crate::model::xmltv::TVGuide; use crate::repository::storage::{short_hash}; use crate::utils::file::file_utils::prepare_file_path; @@ -25,8 +25,12 @@ pub async fn get_xmltv(client: Arc, _cfg: &Config, input: &Conf Some(urls) => { let mut errors = vec![]; let mut file_paths = vec![]; - for url in urls { - match download_epg_file(url, &client, input, working_dir).await { + + for url in match urls { + EpgUrl::Single(value) => vec![value.to_string()], + EpgUrl::Multi(value) => value.clone(), + } { + match download_epg_file(&url, &client, input, working_dir).await { Ok(file_path) => { file_paths.push(file_path); }