diff --git a/Cargo.lock b/Cargo.lock index 5fe912811..da44e7414 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1982,7 +1982,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf763ab1c7a3aa408be466efc86efe35ed1bd3dd74173ed39d6b0d0a6f0ba148" dependencies = [ "memchr", - "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d1ea85f18..8d41df7dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ strip = true serde = { version = "1.0", features = ["derive", "rc"] } serde_yaml = "0.9" serde_json = { version = "1", features = ["arbitrary_precision"] } -quick-xml = { version = "0.37", features = ["serialize"] } +quick-xml = "0.37" regex = "1.11" clap = { version = "4", features = ["derive"] } url = "2.5" diff --git a/docker/Dockerfile b/docker/Dockerfile index 15516a9b2..5bb024159 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,7 +37,12 @@ RUN ffmpeg -loop 1 -i ./resources/channel_unavailable.jpg -t 10 -r 1 -an \ RUN ffmpeg -loop 1 -i ./resources/user_connections_exhausted.jpg -t 10 -r 1 -an \ -vf "scale=1920:1080" \ -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p \ - ./resources/user_connections_exhausted.ts \ + ./resources/user_connections_exhausted.ts + +RUN ffmpeg -loop 1 -i ./resources/provider_connections_exhausted.jpg -t 10 -r 1 -an \ + -vf "scale=1920:1080" \ + -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p \ + ./resources/provider_connections_exhausted.ts # Final container (Scratch) FROM scratch as scratch-final diff --git a/src/model/xmltv.rs b/src/model/xmltv.rs index 78fffe10f..4f93013f8 100644 --- a/src/model/xmltv.rs +++ b/src/model/xmltv.rs @@ -9,6 +9,8 @@ pub const EPG_TAG_PROGRAMME: &str = "programme"; pub const EPG_TAG_CHANNEL: &str = "channel"; pub const EPG_ATTRIB_ID: &str = "id"; pub const EPG_ATTRIB_CHANNEL: &str = "channel"; +pub const EPG_TAG_DISPLAY_NAME: &str = "display-name"; +pub const EPG_TAG_ICON: &str = "icon"; // https://github.com/XMLTV/xmltv/blob/master/xmltv.dtd @@ -17,7 +19,9 @@ pub struct XmlTag { pub name: String, pub value: Option, pub attributes: Option>>, + pub t_attributes: Option>, pub children: Option>>, + pub icon: Option, } impl XmlTag { diff --git a/src/processing/parser/m3u.rs b/src/processing/parser/m3u.rs index 4d4957213..2ab2b67b8 100644 --- a/src/processing/parser/m3u.rs +++ b/src/processing/parser/m3u.rs @@ -1,6 +1,7 @@ use std::borrow::BorrowMut; use crate::model::config::{Config, ConfigInput}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; +use crate::processing::parser::xmltv::normalize_channel_name; use crate::utils::hash_utils::extract_id_from_url; use crate::utils::string_utils; @@ -122,14 +123,18 @@ fn process_header(input: &ConfigInput, video_suffixes: &[&str], content: &str, u } c = it.next(); } + if plih.id.is_empty() { + let channel_id = normalize_channel_name(&plih.name); if let Some(chanid) = extract_id_from_url(url) { plih.id = chanid; + } else { + plih.id = channel_id.to_string(); } + plih.epg_channel_id = Some(channel_id); + } else { + plih.epg_channel_id = Some(plih.id.to_lowercase().to_string()); } - // plih.virtual_id = plih.id; - plih.epg_channel_id = Some(plih.id.to_lowercase().to_string()); - } if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index ed309e4c8..dc8775b2d 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -1,14 +1,33 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use crate::model::xmltv::{Epg, TVGuide, XmlTag, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_CHANNEL, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_PROGRAMME, EPG_TAG_TV}; +use crate::utils::compression::compressed_file_reader::CompressedFileReader; use quick_xml::events::Event; use quick_xml::Reader; +use regex::Regex; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, LazyLock}; +use unidecode::unidecode; -use crate::model::xmltv::{Epg, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_TV, EPG_TAG_CHANNEL, EPG_TAG_PROGRAMME, TVGuide, XmlTag}; -use crate::utils::compression::compressed_file_reader::CompressedFileReader; +static NORMALIZE_CHANNEL: LazyLock = LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9\-]").unwrap()); + +const TERMS_TO_REMOVE: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]; + +pub fn normalize_channel_name(name: &str) -> String { + let normalized = unidecode(name).to_lowercase(); + + // Remove all non-alphanumeric characters (except dashes and underscores). + let cleaned_name = NORMALIZE_CHANNEL.replace_all(&normalized, ""); + + // Remove terms like resolution + let result = TERMS_TO_REMOVE.iter().fold(cleaned_name.to_string(), |acc, term| { + acc.replace(*term, "") + }); + + result +} impl TVGuide { - pub fn filter(&self, channel_ids: &HashSet) -> Option { - if channel_ids.is_empty() { + pub fn filter(&self, epg_channel_ids: &HashSet<&String>) -> Option { + if epg_channel_ids.is_empty() { return None; } match CompressedFileReader::new(&self.file) { @@ -18,10 +37,10 @@ impl TVGuide { let mut filter_tags = |tag: XmlTag| { if match tag.name.as_str() { EPG_TAG_CHANNEL => { - tag.get_attribute_value(EPG_ATTRIB_ID).is_some_and(|val| channel_ids.contains(val)) + tag.get_attribute_value(EPG_ATTRIB_ID).is_some_and(|val| epg_channel_ids.contains(val)) } EPG_TAG_PROGRAMME => { - tag.get_attribute_value(EPG_ATTRIB_CHANNEL).is_some_and(|val| channel_ids.contains(val)) + tag.get_attribute_value(EPG_ATTRIB_CHANNEL).is_some_and(|val| epg_channel_ids.contains(val)) }, EPG_TAG_TV => { tv_attributes.clone_from(&tag.attributes); @@ -80,14 +99,18 @@ where Some((key, value)) } }).collect::>(); - let tag = XmlTag { + let attribs = if attributes.is_empty() { None } else { Some(attributes) }; + let mut tag = XmlTag { name, value: None, - attributes: if attributes.is_empty() { None } else { Some(Arc::new(attributes)) }, + t_attributes: attribs, + attributes: None, children: None, + icon: None, }; if is_tv_tag { + tag.attributes = tag.t_attributes.take().map(|v| Arc::new(v)); callback(tag); } else { stack.push(tag); @@ -95,16 +118,44 @@ where } Ok(Event::End(_e)) => { if !stack.is_empty() { - if let Some(tag) = stack.pop() { + if let Some(mut tag) = stack.pop() { if tag.name == EPG_TAG_CHANNEL { - if let Some(chan_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { + let has_chan_id = tag.get_attribute_value(EPG_ATTRIB_ID).is_some(); + if let Some(children) = &mut tag.children { + for child in children { + match child.name.as_str() { + EPG_TAG_DISPLAY_NAME => { + if !has_chan_id { + if let Some(name) = &child.value { + let norm = normalize_channel_name(name); + if let Some(attribs) = tag.t_attributes.as_mut() { + attribs.insert(EPG_ATTRIB_ID.to_string(), norm); + } else { + tag.t_attributes = Some(HashMap::from([(EPG_ATTRIB_ID.to_string(), norm)])); + } + } + } + } + EPG_TAG_ICON => { + if let Some(src) = child.get_attribute_value("src") { + tag.icon = Some(src.to_string()); + } + } + _ => {} + } + } + } + + if let Some(chan_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { if !chan_id.is_empty() { + tag.attributes = tag.t_attributes.take().map(|v| Arc::new(v)); callback(tag); } - } + } } else if tag.name == EPG_TAG_PROGRAMME { if let Some(chan_id) = tag.get_attribute_value(EPG_ATTRIB_CHANNEL) { if !chan_id.is_empty() { + tag.attributes = tag.t_attributes.take().map(|v| Arc::new(v)); callback(tag); } } @@ -112,8 +163,8 @@ where if let Some(old_tag) = stack.pop().map(|mut r| { let rc_tag = Arc::new(tag); r.children = Some( - r.children.map_or_else(|| vec![rc_tag.clone()], |mut c| { - c.push(rc_tag.clone()); + r.children.map_or_else(|| vec![Arc::clone(&rc_tag)], |mut c| { + c.push(Arc::clone(&rc_tag)); c })); r @@ -175,10 +226,9 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option { #[cfg(test)] mod tests { - use std::collections::HashSet; + use crate::model::xmltv::TVGuide; use std::io; use std::path::PathBuf; - use crate::model::xmltv::{TVGuide}; #[test] fn parse_test() -> io::Result<()> { @@ -188,7 +238,7 @@ mod tests { let tv_guide = TVGuide { file: file_path}; let channel_ids = vec!["channel.1", "channel.2", "channel.3"]; - let channel_ids : HashSet = channel_ids.into_iter().map(|s| s.to_string()).collect(); + let channel_ids = channel_ids.into_iter().map(|s| s).collect(); match tv_guide.filter(&channel_ids) { None => assert!(false, "No epg filtered"), diff --git a/src/processing/parser/xtream.rs b/src/processing/parser/xtream.rs index 41f9a11fd..442ddd513 100644 --- a/src/processing/parser/xtream.rs +++ b/src/processing/parser/xtream.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; use std::sync::Arc; use serde_json::Value; - use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, create_m3u_filter_error_result}; use crate::model::config::ConfigInput; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream}; +use crate::processing::parser::xmltv::normalize_channel_name; use crate::utils::hash_utils::generate_playlist_uuid; use crate::utils::network::xtream::{get_xtream_stream_url_base, ACTION_GET_SERIES_INFO}; @@ -148,7 +148,7 @@ pub fn parse_xtream(input: &ConfigInput, group: category_name.to_string(), title: stream.name.to_string(), url: stream_url.to_string(), - epg_channel_id: stream.epg_channel_id.as_ref().map(|id| id.to_lowercase().to_string()), + epg_channel_id: Some(normalize_channel_name(stream.epg_channel_id.as_ref().map_or_else(|| &stream.name, |id| id))), item_type, xtream_cluster, additional_properties: stream.get_additional_properties(), @@ -180,4 +180,3 @@ pub fn parse_xtream(input: &ConfigInput, Err(err) => Err(err) } } - diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index cf5c80c1f..96ef58c62 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -33,6 +33,8 @@ use crate::repository::playlist_repository::persist_playlist; use crate::utils::default_utils::default_as_default; use crate::utils::{debug_if_enabled}; +use crate::model::xmltv::{EPG_ATTRIB_ID}; + fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool { let provider = ValueProvider { pli }; target.filter(&provider) @@ -536,17 +538,27 @@ async fn process_playlist_for_target(client: Arc, for mut fp in processed_fetched_playlists { // collect all epg_channel ids let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) - .filter_map(|c| c.header.epg_channel_id.clone()).collect(); - - new_playlist.append(&mut fp.playlistgroups); + .filter_map(|c| c.header.epg_channel_id.as_ref()).collect(); if epg_channel_ids.is_empty() { debug_if_enabled!("channel ids are empty"); } else if let Some(tv_guide) = fp.epg { debug!("found epg information for {}", &target.name); if let Some(epg) = tv_guide.filter(&epg_channel_ids) { + let epg_icons: HashMap<&String, &String> = epg.children.iter() + .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) + .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t.icon.as_ref().unwrap())).collect(); + fp.playlistgroups.iter_mut() + .flat_map(|g| &mut g.channels) + .filter(|c| c.header.epg_channel_id.is_some() && c.header.logo == "") + .for_each(|c| { + if let Some(icon) = epg_icons.get(c.header.epg_channel_id.as_ref().unwrap()) { + c.header.logo = icon.to_string(); + } + }); new_epg.push(epg); } } + new_playlist.append(&mut fp.playlistgroups); } if new_playlist.is_empty() { @@ -599,4 +611,3 @@ pub async fn exec_processing(client: Arc, cfg: Arc, tar let elapsed = start_time.elapsed().as_secs(); info!("Update process finished! Took {elapsed} secs."); } -