diff --git a/src/model/xmltv.rs b/src/model/xmltv.rs index 4f93013f8..1a3289248 100644 --- a/src/model/xmltv.rs +++ b/src/model/xmltv.rs @@ -19,9 +19,9 @@ pub struct XmlTag { pub name: String, pub value: Option, pub attributes: Option>>, - pub t_attributes: Option>, pub children: Option>>, pub icon: Option, + pub normalized_epg_id: Option, } impl XmlTag { diff --git a/src/processing/parser/m3u.rs b/src/processing/parser/m3u.rs index 2ab2b67b8..eadd26afc 100644 --- a/src/processing/parser/m3u.rs +++ b/src/processing/parser/m3u.rs @@ -1,7 +1,6 @@ use std::borrow::BorrowMut; use crate::model::config::{Config, ConfigInput}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use crate::processing::parser::xmltv::normalize_channel_name; use crate::utils::hash_utils::extract_id_from_url; use crate::utils::string_utils; @@ -125,15 +124,12 @@ fn process_header(input: &ConfigInput, video_suffixes: &[&str], content: &str, u } if plih.id.is_empty() { - let channel_id = normalize_channel_name(&plih.name); + plih.epg_channel_id = None; if let Some(chanid) = extract_id_from_url(url) { plih.id = chanid; - } else { - plih.id = channel_id.to_string(); } - plih.epg_channel_id = Some(channel_id); } else { - plih.epg_channel_id = Some(plih.id.to_lowercase().to_string()); + plih.epg_channel_id = Some(plih.id.to_string()); } } diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index dc8775b2d..2378f9d2b 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -26,8 +26,8 @@ pub fn normalize_channel_name(name: &str) -> String { } impl TVGuide { - pub fn filter(&self, epg_channel_ids: &HashSet<&String>) -> Option { - if epg_channel_ids.is_empty() { + pub fn filter(&self, epg_channel_ids: &mut HashSet, normalized_epg_channel_ids: &mut HashMap>) -> Option { + if epg_channel_ids.is_empty() && normalized_epg_channel_ids.is_empty() { return None; } match CompressedFileReader::new(&self.file) { @@ -35,21 +35,35 @@ impl TVGuide { let mut children: Vec = vec![]; let mut tv_attributes: Option>> = None; let mut filter_tags = |tag: XmlTag| { - if match tag.name.as_str() { + match tag.name.as_str() { EPG_TAG_CHANNEL => { - tag.get_attribute_value(EPG_ATTRIB_ID).is_some_and(|val| epg_channel_ids.contains(val)) + if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { + if let Some(normalized_epg_id) = tag.normalized_epg_id.as_ref() { + match normalized_epg_channel_ids.entry(normalized_epg_id.to_string()) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + entry.insert(Some(epg_id.to_string())); + epg_channel_ids.insert(epg_id.to_string()); + } + std::collections::hash_map::Entry::Vacant(_entry) => {} + }; + } + if epg_channel_ids.contains(epg_id.as_str()) { + children.push(tag); + } + } } EPG_TAG_PROGRAMME => { - tag.get_attribute_value(EPG_ATTRIB_CHANNEL).is_some_and(|val| epg_channel_ids.contains(val)) - }, + if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_CHANNEL) { + if epg_channel_ids.contains(epg_id.as_str()) { + children.push(tag); + } + } + } EPG_TAG_TV => { tv_attributes.clone_from(&tag.attributes); - false - }, - _ => false, - } { - children.push(tag); - } + } + _ => {} + }; }; parse_tvguide(&mut reader, &mut filter_tags); @@ -80,7 +94,7 @@ where Ok(Event::Eof) => break, Ok(Event::Start(e)) => { let name = String::from_utf8_lossy(e.name().as_ref()).as_ref().to_owned(); - let (is_tv_tag, is_channel, is_program) = match name.as_str() { + let (is_tv_tag, is_channel, is_program) = match name.as_str() { EPG_TAG_TV => (true, false, false), EPG_TAG_CHANNEL => (false, true, false), EPG_TAG_PROGRAMME => (false, false, true), @@ -90,7 +104,7 @@ where .filter_map(|a| { let key = String::from_utf8_lossy(a.key.as_ref()).to_string(); let mut value = String::from(a.unescape_value().unwrap().as_ref()); - if (is_channel && key == EPG_ATTRIB_ID) || (is_program && key == EPG_ATTRIB_CHANNEL) { + if (is_channel && key == EPG_ATTRIB_ID) || (is_program && key == EPG_ATTRIB_CHANNEL) { value = value.to_lowercase().to_string(); } if value.is_empty() { @@ -99,18 +113,17 @@ where Some((key, value)) } }).collect::>(); - let attribs = if attributes.is_empty() { None } else { Some(attributes) }; - let mut tag = XmlTag { + let attribs = if attributes.is_empty() { None } else { Some(Arc::new(attributes)) }; + let tag = XmlTag { name, value: None, - t_attributes: attribs, - attributes: None, + attributes: attribs, children: None, icon: None, + normalized_epg_id: None, }; if is_tv_tag { - tag.attributes = tag.t_attributes.take().map(|v| Arc::new(v)); callback(tag); } else { stack.push(tag); @@ -120,20 +133,12 @@ where if !stack.is_empty() { if let Some(mut tag) = stack.pop() { if tag.name == EPG_TAG_CHANNEL { - let has_chan_id = tag.get_attribute_value(EPG_ATTRIB_ID).is_some(); - if let Some(children) = &mut tag.children { + if let Some(children) = &mut tag.children { for child in children { match child.name.as_str() { EPG_TAG_DISPLAY_NAME => { - if !has_chan_id { - if let Some(name) = &child.value { - let norm = normalize_channel_name(name); - if let Some(attribs) = tag.t_attributes.as_mut() { - attribs.insert(EPG_ATTRIB_ID.to_string(), norm); - } else { - tag.t_attributes = Some(HashMap::from([(EPG_ATTRIB_ID.to_string(), norm)])); - } - } + if let Some(name) = &child.value { + tag.normalized_epg_id = Some(normalize_channel_name(name)); } } EPG_TAG_ICON => { @@ -144,18 +149,16 @@ where _ => {} } } - } + } - if let Some(chan_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { + if let Some(chan_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { if !chan_id.is_empty() { - tag.attributes = tag.t_attributes.take().map(|v| Arc::new(v)); callback(tag); } - } + } } else if tag.name == EPG_TAG_PROGRAMME { if let Some(chan_id) = tag.get_attribute_value(EPG_ATTRIB_CHANNEL) { if !chan_id.is_empty() { - tag.attributes = tag.t_attributes.take().map(|v| Arc::new(v)); callback(tag); } } @@ -164,9 +167,9 @@ where let rc_tag = Arc::new(tag); r.children = Some( r.children.map_or_else(|| vec![Arc::clone(&rc_tag)], |mut c| { - c.push(Arc::clone(&rc_tag)); - c - })); + c.push(Arc::clone(&rc_tag)); + c + })); r }) { stack.push(old_tag); @@ -227,6 +230,8 @@ pub fn flatten_tvguide(tv_guides: &[Epg]) -> Option { #[cfg(test)] mod tests { use crate::model::xmltv::TVGuide; + use crate::processing::parser::xmltv::normalize_channel_name; + use std::collections::{HashMap, HashSet}; use std::io; use std::path::PathBuf; @@ -235,12 +240,11 @@ mod tests { let file_path = PathBuf::from("/tmp/epg.xml.gz"); if file_path.exists() { - let tv_guide = TVGuide { file: file_path}; + let tv_guide = TVGuide { file: file_path }; - let channel_ids = vec!["channel.1", "channel.2", "channel.3"]; - let channel_ids = channel_ids.into_iter().map(|s| s).collect(); - - match tv_guide.filter(&channel_ids) { + let mut channel_ids = HashSet::from(["channel.1".to_string(), "channel.2".to_string(), "channel.3".to_string()]); + let mut nomalized = HashMap::new(); + match tv_guide.filter(&mut channel_ids, &mut nomalized) { None => assert!(false, "No epg filtered"), Some(epg) => { assert_eq!(epg.children.len(), channel_ids.len() * 2, "Epg size does not match") @@ -249,4 +253,9 @@ mod tests { } Ok(()) } + + #[test] + fn normalize() { + assert_eq!("satsupersport6", normalize_channel_name("SAT: SUPERSPORT 6 ᴿᴬᵂ")); + } } \ No newline at end of file diff --git a/src/processing/parser/xtream.rs b/src/processing/parser/xtream.rs index 442ddd513..b30af6e77 100644 --- a/src/processing/parser/xtream.rs +++ b/src/processing/parser/xtream.rs @@ -1,13 +1,12 @@ -use std::collections::HashMap; -use std::sync::Arc; -use serde_json::Value; -use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, create_m3u_filter_error_result}; +use crate::m3u_filter_error::{create_m3u_filter_error_result, M3uFilterError, M3uFilterErrorKind}; use crate::model::config::ConfigInput; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream}; -use crate::processing::parser::xmltv::normalize_channel_name; use crate::utils::hash_utils::generate_playlist_uuid; use crate::utils::network::xtream::{get_xtream_stream_url_base, ACTION_GET_SERIES_INFO}; +use serde_json::Value; +use std::collections::HashMap; +use std::sync::Arc; fn map_to_xtream_category(categories: &Value) -> Result, M3uFilterError> { match serde_json::from_value::>(categories.to_owned()) { @@ -101,8 +100,8 @@ pub fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &st stream: &XtreamStream, live_stream_use_prefix: bool, live_stream_without_extension: bool) -> String { if stream.direct_source.is_empty() { get_xtream_url(xtream_cluster, url, username, password, stream.get_stream_id(), - stream.container_extension.as_ref().map(std::string::ToString::to_string).as_ref(), - live_stream_use_prefix, live_stream_without_extension) + stream.container_extension.as_ref().map(std::string::ToString::to_string).as_ref(), + live_stream_use_prefix, live_stream_without_extension) } else { stream.direct_source.to_string() } @@ -139,6 +138,15 @@ pub fn parse_xtream(input: &ConfigInput, let category_name = &group.category_name; let stream_url = create_xtream_url(xtream_cluster, url, username, password, stream, live_stream_use_prefix, live_stream_without_extension); let item_type = PlaylistItemType::from(xtream_cluster); + // EPG Channel id fix, remove empty + stream.epg_channel_id = if let XtreamCluster::Live = xtream_cluster { + stream.epg_channel_id.as_ref() + .filter(|epg_id| !epg_id.trim().is_empty()) + .map(|epg_id| epg_id.to_lowercase()) + .or(None) + } else { + None + }; let item = PlaylistItem { header: PlaylistItemHeader { id: stream.get_stream_id().to_string(), @@ -148,7 +156,7 @@ pub fn parse_xtream(input: &ConfigInput, group: category_name.to_string(), title: stream.name.to_string(), url: stream_url.to_string(), - epg_channel_id: Some(normalize_channel_name(stream.epg_channel_id.as_ref().map_or_else(|| &stream.name, |id| id))), + epg_channel_id: stream.epg_channel_id.clone(), item_type, xtream_cluster, additional_properties: stream.get_additional_properties(), diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index 96ef58c62..3eb779498 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -26,7 +26,7 @@ use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor use crate::model::stats::{InputStats, PlaylistStats, SourceStats, TargetStats}; use crate::processing::processor::affix::apply_affixes; use crate::processing::playlist_watch::process_group_watch; -use crate::processing::parser::xmltv::flatten_tvguide; +use crate::processing::parser::xmltv::{flatten_tvguide, normalize_channel_name}; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::persist_playlist; @@ -537,22 +537,43 @@ async fn process_playlist_for_target(client: Arc, // we need to process each input epg. for mut fp in processed_fetched_playlists { // collect all epg_channel ids - let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) - .filter_map(|c| c.header.epg_channel_id.as_ref()).collect(); - if epg_channel_ids.is_empty() { - debug_if_enabled!("channel ids are empty"); + let mut epg_channel_ids: HashSet = HashSet::new(); + let mut normalized_epg_channel_ids: HashMap> = HashMap::new(); + for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { + match channel.header.epg_channel_id.as_ref() { + None => {normalized_epg_channel_ids.insert(normalize_channel_name(&channel.header.name), None);}, + Some(epg_id) => {epg_channel_ids.insert(epg_id.to_string());}, + }; + } + // let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) + // .filter_map(|c| c.header.epg_channel_id.as_ref()).map(|a| a.as_str()).collect(); + if epg_channel_ids.is_empty() && normalized_epg_channel_ids.is_empty() { + debug!("channel ids are empty"); } else if let Some(tv_guide) = fp.epg { - debug!("found epg information for {}", &target.name); - if let Some(epg) = tv_guide.filter(&epg_channel_ids) { + debug_if_enabled!("found epg information for {}", &target.name); + if let Some(epg) = tv_guide.filter(&mut epg_channel_ids, &mut normalized_epg_channel_ids) { let epg_icons: HashMap<&String, &String> = epg.children.iter() .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t.icon.as_ref().unwrap())).collect(); fp.playlistgroups.iter_mut() .flat_map(|g| &mut g.channels) - .filter(|c| c.header.epg_channel_id.is_some() && c.header.logo == "") + .filter(|c| c.header.xtream_cluster == XtreamCluster::Live ) + .filter(|c| c.header.epg_channel_id.is_none() || c.header.logo.is_empty() || c.header.logo_small.is_empty()) .for_each(|c| { - if let Some(icon) = epg_icons.get(c.header.epg_channel_id.as_ref().unwrap()) { - c.header.logo = icon.to_string(); + if c.header.epg_channel_id.as_ref().is_none() { + if let Some(Some(epg_id)) = normalized_epg_channel_ids.get(&normalize_channel_name(&c.header.name)) { + c.header.epg_channel_id = Some(epg_id.to_string()); + } + } + if c.header.epg_channel_id.is_some() && (c.header.logo.is_empty() || c.header.logo_small.is_empty()) { + if let Some(icon) = epg_icons.get(c.header.epg_channel_id.as_ref().unwrap()) { + if c.header.logo.is_empty() { + c.header.logo = icon.to_string(); + } + if c.header.logo_small.is_empty() { + c.header.logo = icon.to_string(); + } + } } }); new_epg.push(epg); diff --git a/src/repository/indexed_document.rs b/src/repository/indexed_document.rs index a3cf7afa7..07da4af7e 100644 --- a/src/repository/indexed_document.rs +++ b/src/repository/indexed_document.rs @@ -512,8 +512,10 @@ mod tests { use std::path::PathBuf; use serde::{Deserialize, Serialize}; + use crate::model::playlist::XtreamPlaylistItem; // use crate::model::playlist::XtreamPlaylistItem; use crate::repository::indexed_document::{IndexedDocumentGarbageCollector, IndexedDocumentIterator, IndexedDocumentWriter}; + use crate::utils::file::config_reader::resolve_env_var; // Example usage with a simple struct #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -585,14 +587,16 @@ mod tests { Ok(()) } - // #[test] - // fn test_read_xt() -> io::Result<()> { - // let main_path = PathBuf::from("../m3u-test/settings/m3u-silver/data/xt_m3u/xtream/live.db"); - // let index_path = PathBuf::from("../m3u-test/settings/m3u-silver/data/xt_m3u/xtream/live.idx"); - // let reader = IndexedDocumentIterator::::new(&main_path, &index_path)?; - // for doc in reader { - // println!("{doc:?}"); - // } - // Ok(()) - // } + #[test] + fn test_read_xt() -> io::Result<()> { + let main_path = PathBuf::from(resolve_env_var("${env:HOME}/projects/m3u-test/settings/alexyand/data/all_channels/xtream/live.db")); + let index_path = PathBuf::from(resolve_env_var("${env:HOME}/projects/m3u-test/settings/alexyand/data/all_channels/xtream/live.idx")); + let reader = IndexedDocumentIterator::::new(&main_path, &index_path)?; + for (doc, _has_next) in reader { + if doc.epg_channel_id.is_none() { + println!("{doc:?}"); + } + } + Ok(()) + } }