From 28db24f62e8cf3b0f17ee5d531e768279d202a26 Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:46:12 +0300 Subject: [PATCH 1/7] Update Cargo.toml --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d1ea85f18..67ae4ace2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ strip = true serde = { version = "1.0", features = ["derive", "rc"] } serde_yaml = "0.9" serde_json = { version = "1", features = ["arbitrary_precision"] } -quick-xml = { version = "0.37", features = ["serialize"] } +quick-xml = "0.37" regex = "1.11" clap = { version = "4", features = ["derive"] } url = "2.5" @@ -54,6 +54,8 @@ paste = "1.0" tempfile = "3.19" ruzstd = "0" filetime = "0.2" +unicode-normalization = "0.1" +deunicode = "1.6" #[cfg(target_os = "macos")] libc = "0" #[cfg(target_os = "windows")] From b63dfb0ddc0f973899acfe16addfa0b5301e947d Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:49:52 +0300 Subject: [PATCH 2/7] Update Dockerfile --- docker/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 15516a9b2..5bb024159 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,7 +37,12 @@ RUN ffmpeg -loop 1 -i ./resources/channel_unavailable.jpg -t 10 -r 1 -an \ RUN ffmpeg -loop 1 -i ./resources/user_connections_exhausted.jpg -t 10 -r 1 -an \ -vf "scale=1920:1080" \ -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p \ - ./resources/user_connections_exhausted.ts \ + ./resources/user_connections_exhausted.ts + +RUN ffmpeg -loop 1 -i ./resources/provider_connections_exhausted.jpg -t 10 -r 1 -an \ + -vf "scale=1920:1080" \ + -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p \ + ./resources/provider_connections_exhausted.ts # Final container (Scratch) FROM scratch as scratch-final From b63ee124a5258a34a7401b88474c25b1a49ca50d Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:53:47 +0300 Subject: [PATCH 3/7] Update epg.rs --- src/utils/network/epg.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/utils/network/epg.rs b/src/utils/network/epg.rs index cbea69a25..dbdd0b746 100644 --- a/src/utils/network/epg.rs +++ b/src/utils/network/epg.rs @@ -1,6 +1,10 @@ use std::sync::Arc; use log::debug; -use crate::m3u_filter_error::M3uFilterError; +use std::fs::File; +use std::io::BufReader; + +use crate::model::xmltv::{Epg, XmlTag}; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigInput}; use crate::model::xmltv::TVGuide; use crate::utils::file::file_utils::prepare_file_path; @@ -23,4 +27,18 @@ pub async fn get_xmltv(client: Arc, _cfg: &Config, input: &Conf } } } -} \ No newline at end of file +} + + +pub fn parse_epg(path: &std::path::Path) -> Result { + let file = File::open(path).map_err(|e| { + M3uFilterError::new(M3uFilterErrorKind::Notify, format!("Failed to parse EPG: {}", e)) + //M3uFilterError::new(M3uFilterErrorKind::IOError, format!("Failed to open EPG file: {}", e)) + })?; + let reader = BufReader::new(file); + + XmlTag::parse_root(reader).map_err(|e| { + M3uFilterError::new(M3uFilterErrorKind::Notify, format!("Failed to parse XMLTV: {}", e)) + //M3uFilterError::new(M3uFilterErrorKind::ParseError, format!("Failed to parse XMLTV: {}", e)) + }) +} From f21a99703734bfd546484d1719e88ba5d67995c2 Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:56:57 +0300 Subject: [PATCH 4/7] Update xmltv.rs --- src/model/xmltv.rs | 131 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/src/model/xmltv.rs b/src/model/xmltv.rs index 78fffe10f..5176496ff 100644 --- a/src/model/xmltv.rs +++ b/src/model/xmltv.rs @@ -1,8 +1,10 @@ use std::collections::{HashMap}; use std::path::PathBuf; use std::sync::Arc; -use quick_xml::{Error, Writer}; +use quick_xml::{Writer}; use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; +use quick_xml::Error; +use quick_xml::Reader; pub const EPG_TAG_TV: &str = "tv"; pub const EPG_TAG_PROGRAMME: &str = "programme"; @@ -39,6 +41,133 @@ impl XmlTag { } Ok(writer.write_event(Event::End(BytesEnd::new(self.name.as_str())))?) } + + pub fn parse_root(mut reader: R) -> Result { + let mut xml = Reader::from_reader(&mut reader); + //xml.trim_text(true); + let mut buf = Vec::new(); + + let _children: Vec = Vec::new(); + while let Ok(event) = xml.read_event_into(&mut buf) { + match event { + Event::Start(start) if start.name().as_ref() == b"tv" => { + // Parse tag + let epg_children = Self::parse_children(&mut xml)?; + return Ok(Epg { + attributes: None, + children: epg_children, + }); + } + Event::Eof => break, + _ => {} + } + buf.clear(); + } + + + Err(quick_xml::Error::Io(std::sync::Arc::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Expected root", + )))) + } + + fn parse_children(xml: &mut Reader) -> Result, quick_xml::Error> { + let mut children = Vec::new(); + let mut buf = Vec::new(); + + loop { + match xml.read_event_into(&mut buf)? { + Event::Start(e) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + let mut text = None; + let mut tag_buf = Vec::new(); + let mut sub_children = vec![]; + + loop { + match xml.read_event_into(&mut tag_buf)? { + Event::Text(t) => { + text = Some(t.unescape()?.to_string()); + } + Event::Start(sub_start) => { + let sub_name = String::from_utf8_lossy(sub_start.name().as_ref()).to_string(); + let mut sub_text = None; + let mut inner_buf = Vec::new(); + let mut inner_children = vec![]; + + loop { + match xml.read_event_into(&mut inner_buf)? { + Event::Text(t) => { + sub_text = Some(t.unescape()?.to_string()); + } + Event::Start(_) => { + let inner = Self::parse_children(xml)?; + inner_children.extend(inner); + } + Event::End(end) if end.name().as_ref() == sub_start.name().as_ref() => break, + Event::Eof => break, + _ => {} + } + inner_buf.clear(); + } + + sub_children.push(XmlTag { + name: sub_name, + value: sub_text, + attributes: Some(Arc::new({ + let mut map = HashMap::new(); + for attr in sub_start.attributes().with_checks(false) { + if let Ok(attr) = attr { + let key = String::from_utf8_lossy(attr.key.as_ref()).to_string(); + let val = attr.unescape_value()?.to_string(); + map.insert(key, val); + } + } + map + })), + children: if inner_children.is_empty() { + None + } else { + Some(inner_children.into_iter().map(Arc::new).collect()) + }, + }); + } + Event::End(end) if end.name().as_ref() == e.name().as_ref() => break, + Event::Eof => break, + _ => {} + } + tag_buf.clear(); + } + + children.push(XmlTag { + name, + value: text, + attributes: Some(Arc::new({ + let mut map = HashMap::new(); + for attr in e.attributes().with_checks(false) { + if let Ok(attr) = attr { + let key = String::from_utf8_lossy(attr.key.as_ref()).to_string(); + let val = attr.unescape_value()?.to_string(); + map.insert(key, val); + } + } + map + })), + children: if sub_children.is_empty() { + None + } else { + Some(sub_children.into_iter().map(Arc::new).collect()) + }, + }); + } + Event::End(_) | Event::Eof => break, + _ => {} + } + + buf.clear(); + } + + Ok(children) + } } From ad0c161883f6bc354a70cd37c5a024eb5300a408 Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:59:08 +0300 Subject: [PATCH 5/7] Update playlist.rs --- src/model/playlist.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/model/playlist.rs b/src/model/playlist.rs index 64e4d6429..2167dfe1b 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -159,6 +159,7 @@ pub struct PlaylistItemHeader { #[serde(default)] pub category_id: u32, pub input_name: String, + pub stream_icon: Option, } impl PlaylistItemHeader { @@ -618,4 +619,4 @@ impl PlaylistGroup { self.channels.iter().filter(|&c| filter(c)).count() } -} \ No newline at end of file +} From cae368c1d8a3a247067fba929bc2f0d99f291d90 Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:01:57 +0300 Subject: [PATCH 6/7] Update xtream.rs --- src/processing/parser/xtream.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/processing/parser/xtream.rs b/src/processing/parser/xtream.rs index 41f9a11fd..0035ec986 100644 --- a/src/processing/parser/xtream.rs +++ b/src/processing/parser/xtream.rs @@ -2,6 +2,8 @@ use std::collections::HashMap; use std::sync::Arc; use serde_json::Value; +use deunicode::deunicode; + use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, create_m3u_filter_error_result}; use crate::model::config::ConfigInput; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; @@ -148,7 +150,8 @@ pub fn parse_xtream(input: &ConfigInput, group: category_name.to_string(), title: stream.name.to_string(), url: stream_url.to_string(), - epg_channel_id: stream.epg_channel_id.as_ref().map(|id| id.to_lowercase().to_string()), + epg_channel_id: stream.epg_channel_id.as_ref().map(|id| deunicode(id.to_lowercase().to_string().as_str())), + //epg_channel_id: stream.epg_channel_id.as_ref().map(|id| id.to_lowercase().to_string()), item_type, xtream_cluster, additional_properties: stream.get_additional_properties(), @@ -180,4 +183,3 @@ pub fn parse_xtream(input: &ConfigInput, Err(err) => Err(err) } } - From b9a1632cadbaa8d8adde3864690e756a628b1e23 Mon Sep 17 00:00:00 2001 From: Alex <73880463+alexeyand@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:03:59 +0300 Subject: [PATCH 7/7] Update playlist.rs --- src/processing/processor/playlist.rs | 106 ++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index cf5c80c1f..0d3eb7b35 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -16,6 +16,8 @@ use log::{debug, error, info, log_enabled, trace, warn, Level}; use std::time::Instant; use unidecode::unidecode; +use unicode_normalization::UnicodeNormalization; + use crate::foundation::filter::{get_field_value, set_field_value, MockValueProcessor, ValueProvider}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, get_errors_notify_message, notify_err}; use crate::messaging::{send_message, MsgKind}; @@ -33,6 +35,9 @@ use crate::repository::playlist_repository::persist_playlist; use crate::utils::default_utils::default_as_default; use crate::utils::{debug_if_enabled}; +use crate::model::xmltv::{TVGuide}; +use crate::utils::network::epg::parse_epg; + fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool { let provider = ValueProvider { pli }; target.filter(&provider) @@ -513,6 +518,9 @@ async fn process_playlist_for_target(client: Arc, let mut processed_fetched_playlists: Vec = vec![]; for provider_fpl in playlists.iter_mut() { let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); + + auto_assign_epg_channel_ids(&mut processed_fpl.playlistgroups, provider_fpl.epg.as_ref()); + playlist_resolve_series(Arc::clone(&client), cfg, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; playlist_resolve_vod(Arc::clone(&client), cfg, target, errors, &mut processed_fpl).await; // stats @@ -562,6 +570,103 @@ async fn process_playlist_for_target(client: Arc, } } + +fn auto_assign_epg_channel_ids(groups: &mut [PlaylistGroup], xmltv: Option<&TVGuide>) { + use std::collections::HashMap; + + let Some(tv) = xmltv else { + warn!("EPG: TVGuide is None"); + return; + }; + + let epg_path = &tv.file; + + match parse_epg(epg_path) { + Ok(parsed_epg) => { + let mut epg_names: Vec<(String, String)> = vec![]; + let mut epg_icons: HashMap> = HashMap::new(); + + for tag in &parsed_epg.children { + if tag.name == "channel" { + let Some(id) = tag.get_attribute_value("id").cloned() else { + continue; + }; + + let mut icon_url: Option = None; + + if let Some(children) = &tag.children { + for child in children { + let child = child.as_ref(); + match child.name.as_str() { + "display-name" => { + if let Some(name) = &child.value { + let norm = normalize_channel_name(name); + epg_names.push((norm.clone(), id.clone())); + } + } + "icon" => { + if let Some(src) = child.get_attribute_value("src") { + icon_url = Some(src.clone()); + } + } + _ => {} + } + } + } + + epg_icons.insert(id.clone(), icon_url); + } + } + + for group in groups.iter_mut() { + for chan in group.channels.iter_mut() { + let epg_id_empty = chan.header.epg_channel_id + .as_deref() + .map_or(true, |v| v.trim().is_empty()); + + if epg_id_empty { + let chan_name_raw = &chan.header.name; + let chan_name_norm = normalize_channel_name(chan_name_raw); + + if let Some((_, epg_id)) = epg_names.iter().find(|(k, _)| k == &chan_name_norm) { + chan.header.epg_channel_id = Some(epg_id.clone()); + if let Some(icon) = epg_icons.get(epg_id).and_then(|i| i.clone()) { + chan.header.stream_icon = Some(icon); + } + info!("Exact match '{}' → '{}'", chan_name_raw, epg_id); + } else if let Some((match_key, epg_id)) = epg_names.iter().find(|(k, _)| k.contains(&chan_name_norm)) { + chan.header.epg_channel_id = Some(epg_id.clone()); + if let Some(icon) = epg_icons.get(epg_id).and_then(|i| i.clone()) { + chan.header.stream_icon = Some(icon); + } + info!("Fallback match '{}' → '{}' (from '{}')", chan_name_raw, epg_id, match_key); + } + } + } + } + } + Err(e) => { + error!("Failed to parse EPG: {:?}", e); + } + } + + fn normalize_channel_name(name: &str) -> String { + name.nfkd() + .collect::() + .to_lowercase() + .replace(|c: char| !c.is_alphanumeric(), "") + .replace("3840p", "") + .replace("uhd", "") + .replace("fhd", "") + .replace("hd", "") + .replace("sd", "") + .replace("4k", "") + .replace("plus", "") + .replace("raw", "") + } +} + + fn process_watch(target: &ConfigTarget, cfg: &Config, new_playlist: &Vec) { if target.t_watch_re.is_some() { if default_as_default().eq_ignore_ascii_case(&target.name) { @@ -599,4 +704,3 @@ pub async fn exec_processing(client: Arc, cfg: Arc, tar let elapsed = start_time.elapsed().as_secs(); info!("Update process finished! Took {elapsed} secs."); } -