diff --git a/src/auth/mod.rs b/src/auth/mod.rs index 43b55ba77..9dcb488e3 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -2,4 +2,5 @@ pub mod authenticator; pub mod password; pub mod user; pub mod auth_bearer; -pub mod access_token; \ No newline at end of file +pub mod auth_basic; +pub mod access_token; diff --git a/src/model/xmltv.rs b/src/model/xmltv.rs index dc7f2bd1f..6ab17ad37 100644 --- a/src/model/xmltv.rs +++ b/src/model/xmltv.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap}; use std::path::PathBuf; use quick_xml::{Error, Writer}; use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; @@ -20,7 +20,7 @@ pub struct XmlTag { pub attributes: Option>, pub children: Option>, pub icon: Option, - pub normalized_epg_ids: HashSet, + pub normalized_epg_ids: Vec, } impl XmlTag { @@ -32,7 +32,7 @@ impl XmlTag { attributes: attribs, children: None, icon: None, - normalized_epg_ids: HashSet::new(), + normalized_epg_ids: Vec::new(), } } diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index 5b8e93897..63cd4c76e 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -1,6 +1,5 @@ use crate::model::config::{EpgNamePrefix, EpgSmartMatchConfig}; use crate::model::xmltv::{Epg, TVGuide, XmlTag, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_CHANNEL, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_PROGRAMME, EPG_TAG_TV}; -use crate::processing::processor::playlist::EpgIdCache; use crate::utils::compression::compressed_file_reader::CompressedFileReader; use deunicode::deunicode; use quick_xml::events::{BytesStart, BytesText, Event}; @@ -13,6 +12,7 @@ use std::mem; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, RwLock}; +use crate::processing::processor::epg::EpgIdCache; use crate::utils::constants::CONSTANTS; fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a str>, &'a str) { @@ -92,7 +92,7 @@ impl TVGuide { EPG_TAG_DISPLAY_NAME => { if smart_match { if let Some(name) = &child.value { - tag.normalized_epg_ids.insert(normalize_channel_name(name, &id_cache.smart_match_config)); + tag.normalized_epg_ids.push(normalize_channel_name(name, &id_cache.smart_match_config)); } } } @@ -108,33 +108,19 @@ impl TVGuide { } fn try_fuzzy_matching(id_cache: &mut EpgIdCache, epg_id: &str, tag: &XmlTag, fuzzy_matching: bool) -> bool { - let id: Cow = Cow::Owned(epg_id.to_string()); - let mut matched = { - let mut match_found = false; - for normalized_epg_id in &tag.normalized_epg_ids { - let key = Cow::Owned(normalized_epg_id.to_string()); - id_cache.normalized.entry(key).and_modify(|entry| { - entry.1 = Some(id.clone()); - id_cache.channel.insert(id.clone()); - match_found = true; - }); - } - match_found - }; - + let mut matched = id_cache.match_with_normalized(epg_id, &tag.normalized_epg_ids); if !matched && fuzzy_matching { - let (fuzzy_matched, matched_normalized_epg_id) = Self::find_best_fuzzy_match(id_cache, tag); - matched = fuzzy_matched; - if matched { - let key = Cow::Owned(matched_normalized_epg_id.unwrap().to_string()); + let (fuzzy_matched, matched_normalized_name) = Self::find_best_fuzzy_match(id_cache, tag); + if fuzzy_matched { + let key = matched_normalized_name.unwrap(); + let id = epg_id.to_string(); id_cache.normalized.entry(key).and_modify(|entry| { - entry.1 = Some(id.clone()); - id_cache.channel.insert(id.clone()); + entry.replace(id.clone()); + id_cache.channel_epg_id.insert(Cow::Owned(id)); matched = true; }); } } - matched } @@ -145,28 +131,26 @@ impl TVGuide { let match_threshold = id_cache.smart_match_config.match_threshold; let best_match_threshold = id_cache.smart_match_config.best_match_threshold; - id_cache.normalized.par_iter().for_each(|(norm_key, (phonetic_code_opt, _))| { + id_cache.normalized.par_iter().for_each(|(norm_key, _epg_id_opt)| { if early_exit_flag.load(Ordering::SeqCst) { return; } - if let Some(phonetic_code) = phonetic_code_opt { - for normalized_epg_id in &tag.normalized_epg_ids { - let code_key: Cow = Cow::Owned(normalized_epg_id.to_string()); - let code = id_cache.phonetic(&code_key); - if &code == phonetic_code { - let match_jw = strsim::jaro_winkler(norm_key, normalized_epg_id); - #[allow(clippy::cast_possible_truncation)] - #[allow(clippy::cast_sign_loss)] - let mjw = min(100, match_jw as u16 * 100); - if mjw >= match_threshold { - let mut lock = data.lock().unwrap(); - if lock.0 < mjw { - *lock = (mjw, Some(Cow::Borrowed(norm_key))); - } - if mjw > best_match_threshold { - early_exit_flag.store(true, Ordering::SeqCst); - return; // (true, matched_normalized_epg_id.map(|s| s.to_string())); - } + let phonetic_code = id_cache.phonetic(norm_key); + for normalized_epg_id in &tag.normalized_epg_ids { + let code = id_cache.phonetic(normalized_epg_id); + if code == phonetic_code { + let match_jw = strsim::jaro_winkler(norm_key, normalized_epg_id); + #[allow(clippy::cast_possible_truncation)] + #[allow(clippy::cast_sign_loss)] + let mjw = min(100, match_jw as u16 * 100); + if mjw >= match_threshold { + let mut lock = data.lock().unwrap(); + if lock.0 < mjw { + *lock = (mjw, Some(Cow::Borrowed(norm_key))); + } + if mjw > best_match_threshold { + early_exit_flag.store(true, Ordering::SeqCst); + return; // (true, matched_normalized_epg_id.map(|s| s.to_string())); } } } @@ -199,7 +183,7 @@ impl TVGuide { } } else { let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); - if id_cache.channel.contains(&borrowed_epg_id) { + if id_cache.channel_epg_id.contains(&borrowed_epg_id) { children.push(tag); } } @@ -210,7 +194,7 @@ impl TVGuide { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_CHANNEL) { if !id_cache.processed.contains(epg_id) { let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); - if id_cache.channel.contains(&borrowed_epg_id) { + if id_cache.channel_epg_id.contains(&borrowed_epg_id) { children.push(tag); } } @@ -245,7 +229,7 @@ impl TVGuide { } pub fn filter(&self, id_cache: &mut EpgIdCache) -> Option { - if id_cache.channel.is_empty() && id_cache.normalized.is_empty() { + if id_cache.channel_epg_id.is_empty() && id_cache.normalized.is_empty() { return None; } let epgs: Vec = self.file_paths.iter() diff --git a/src/processing/processor/epg.rs b/src/processing/processor/epg.rs new file mode 100644 index 000000000..6764e590c --- /dev/null +++ b/src/processing/processor/epg.rs @@ -0,0 +1,182 @@ +use crate::model::config::{EpgConfig, EpgSmartMatchConfig}; +use crate::model::playlist::{FetchedPlaylist, PlaylistItem, XtreamCluster}; +use crate::model::xmltv::Epg; +use crate::processing::parser::xmltv::normalize_channel_name; +use log::debug; +use rphonetic::{DoubleMetaphone, Encoder}; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +pub struct EpgIdCache<'a> { + pub channel_epg_id: HashSet>, + pub normalized: HashMap>, + pub processed: HashSet, + pub smart_match_config: EpgSmartMatchConfig, + pub metaphone: DoubleMetaphone, + pub smart_match_enabled: bool, + pub fuzzy_match_enabled: bool, +} + +impl EpgIdCache<'_> { + pub fn new(epg_config: Option<&EpgConfig>) -> Self { + let normalize_config = epg_config.map_or_else(EpgSmartMatchConfig::default, |epg_config| epg_config.t_smart_match.clone()); + EpgIdCache { + channel_epg_id: HashSet::new(), // contains the epg_ids collected from playlist channels + normalized: HashMap::new(), + processed: HashSet::new(), + metaphone: DoubleMetaphone::default(), + smart_match_enabled: normalize_config.enabled, + fuzzy_match_enabled: normalize_config.enabled && normalize_config.fuzzy_matching, + smart_match_config: normalize_config, + + } + } + + fn is_empty(&self) -> bool { + self.channel_epg_id.is_empty() && self.normalized.is_empty() + } + + fn normalize_and_store(&mut self, name: &str, epg_id: Option<&String>) { + let normalized_name = self.normalize(name); + self.normalized.insert(normalized_name, epg_id.map(|v| v.to_string())); + } + + fn normalize(&self, name: &str) -> String { + normalize_channel_name(name, &self.smart_match_config) + } + + pub(crate) fn phonetic(&self, name: &str) -> String { + self.metaphone.encode(name) + } + + pub fn collect_epg_id(&mut self, fp: &mut FetchedPlaylist) { + let smart_match_enabled = self.smart_match_enabled; + let fuzzy_matching = self.fuzzy_match_enabled; + + for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { + let mut missing_epg_id = true; + // insert epg_id to known channel epg_ids + if let Some(id) = channel.header.epg_channel_id.as_deref() { + if !id.is_empty() { + missing_epg_id = false; + self.channel_epg_id.insert(Cow::Owned(id.to_string())); + } + } + + // for fuzzy_matching we need to put the normalized name even if there is an epg_id, because the epg_id + // could not match to the epg file. And then we try to guess it based on normalized name + let needs_normalization = smart_match_enabled && (fuzzy_matching || missing_epg_id); + + if needs_normalization { + let name = &channel.header.name; + self.normalize_and_store(name, channel.header.epg_channel_id.as_ref()); + } + } + } + + pub fn match_with_normalized(&mut self, epg_id: &str, normalized_epg_ids: &[String]) -> bool { + for key in normalized_epg_ids { + if let Some(entry) = self.normalized.get_mut(key) { + entry.replace(epg_id.to_string()); + self.channel_epg_id.insert(epg_id.to_string().into()); + return true; + } + } + false + } +} + +fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache) { + if let Some(tv_guide) = &fp.epg { + if let Some(epg) = tv_guide.filter(id_cache) { + // // icon tags + // let icon_tags: HashMap<&String, &XmlTag> = epg.children.iter() + // .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) + // .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t)).collect(); + + let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live; + // let filter_missing_epg_id = |chan: &mut PlaylistItem| chan.header.epg_channel_id.is_none() || chan.header.logo.is_empty() || chan.header.logo_small.is_empty(); + let filter_missing_epg_id = |chan: &&mut PlaylistItem| chan.header.epg_channel_id.is_none(); + + let assign_values = |chan: &mut PlaylistItem| { + if id_cache.smart_match_enabled { + // if the channel has no epg_id or the epg_id is not present in xmltv/tvguide then we need to match one from existing tvguide + let not_processed = match &chan.header.epg_channel_id { + None => true, + Some(epg_id) => !id_cache.processed.contains(epg_id), + }; + if not_processed { + let normalized = id_cache.normalize(&chan.header.name); + if let Some(epg_id) = id_cache.normalized.get(&normalized) { + chan.header.epg_channel_id = epg_id.clone(); + } + } + } + // if chan.header.epg_channel_id.is_some() && (chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) { + // if let Some(icon_tag) = icon_tags.get(chan.header.epg_channel_id.as_ref().unwrap()) { + // if let Some(icon) = icon_tag.icon.as_ref() { + // if chan.header.logo.is_empty() { + // chan.header.logo = (*icon).to_string(); + // } + // if chan.header.logo_small.is_empty() { + // chan.header.logo = (*icon).to_string(); + // } + // } + // } + // } + }; + + fp.playlistgroups.iter_mut() + .flat_map(|g| &mut g.channels) + .filter(filter_live) + .filter(filter_missing_epg_id) + .for_each(assign_values); + new_epg.push(epg); + } + } +} + +pub fn process_playlist_epg(fp: &mut FetchedPlaylist, epg: &mut Vec) { + // collect all epg_channel ids + let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); + id_cache.collect_epg_id(fp); + + if id_cache.is_empty() { // TODO && not fuzzy matching { + debug!("No epg ids found"); + } else { + assign_channel_epg(epg, fp, &mut id_cache); + } +} + + +#[cfg(test)] +mod tests { + use rand::distr::Alphanumeric; + use rand::Rng; + use rphonetic::{DoubleMetaphone, Encoder}; + use tokio::time::Instant; + + fn random_string() -> String { + rand::rng() + .sample_iter(&Alphanumeric) + .take(30) + .map(char::from) + .collect() + } + + #[test] + fn test_phonetic() { + let strings: Vec = (0..5_000_000) + .map(|_| random_string()) + .collect(); + + let phonetic = DoubleMetaphone::new(Some(6)); + + let now = Instant::now(); + for value in &strings { + let _ = phonetic.encode(value); + } + + let elapsed = now.elapsed(); + println!("Elapsed time: {}.{:03} secs", elapsed.as_secs(), elapsed.subsec_millis()); + } +} \ No newline at end of file diff --git a/src/processing/processor/mod.rs b/src/processing/processor/mod.rs index 5760394b3..ff4018e7b 100644 --- a/src/processing/processor/mod.rs +++ b/src/processing/processor/mod.rs @@ -3,6 +3,7 @@ mod xtream; mod affix; mod xtream_vod; mod xtream_series; +pub mod epg; #[macro_export] macro_rules! handle_error { diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index dd3fc5b3d..2263295d4 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -1,10 +1,9 @@ -use crate::Config; -use crate::model::config::{ConfigInput, ConfigRename, EpgConfig, EpgSmartMatchConfig}; +use crate::{Config}; +use crate::model::config::{ConfigInput, ConfigRename}; use crate::utils::network::epg; use crate::utils::network::m3u; use crate::utils::network::xtream; use core::cmp::Ordering; -use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::{Arc}; @@ -14,7 +13,6 @@ use std::thread; use log::{debug, error, info, log_enabled, trace, warn, Level}; use std::time::Instant; use deunicode::deunicode; -use rphonetic::{Encoder, Metaphone}; use crate::foundation::filter::{get_field_value, set_field_value, MockValueProcessor, ValueProvider}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind, get_errors_notify_message, notify_err}; use crate::messaging::{send_message, MsgKind}; @@ -25,14 +23,16 @@ use crate::model::playlist::{FetchedPlaylist, FieldGetAccessor, FieldSetAccessor use crate::model::stats::{InputStats, PlaylistStats, SourceStats, TargetStats}; use crate::processing::processor::affix::apply_affixes; use crate::processing::playlist_watch::process_group_watch; -use crate::processing::parser::xmltv::{flatten_tvguide, normalize_channel_name}; use crate::processing::processor::xtream_series::playlist_resolve_series; use crate::processing::processor::xtream_vod::playlist_resolve_vod; use crate::repository::playlist_repository::persist_playlist; use crate::utils::default_utils::default_as_default; use crate::utils::{debug_if_enabled}; -use crate::model::xmltv::{Epg, XmlTag, EPG_ATTRIB_ID}; +use crate::model::xmltv::{Epg}; +use crate::processing::parser::xmltv::flatten_tvguide; +use crate::processing::processor::epg::process_playlist_epg; +use crate::utils::step_measure::StepMeasure; fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool { let provider = ValueProvider { pli }; @@ -510,144 +510,6 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { sort_order } -type PhoneticCodeAndMaybeEpgId<'a> = (Option>, Option>); -pub struct EpgIdCache<'a > { - pub channel: HashSet>, - pub normalized: HashMap, PhoneticCodeAndMaybeEpgId<'a>>, - pub normalized_phonetic: HashMap, Cow<'a, str>>, - pub processed: HashSet, - pub smart_match_config: EpgSmartMatchConfig, - pub metaphone: Metaphone, - pub smart_match_enabled: bool, - pub fuzzy_match_enabled: bool, -} - -impl EpgIdCache<'_> { - pub fn new(epg_config: Option<&EpgConfig>) -> Self { - let normalize_config = epg_config.map_or_else(EpgSmartMatchConfig::default, |epg_config| epg_config.t_smart_match.clone()); - EpgIdCache { - channel: HashSet::new(), - normalized: HashMap::new(), - normalized_phonetic: HashMap::new(), - processed: HashSet::new(), - metaphone: Metaphone::default(), - smart_match_enabled: normalize_config.enabled, - fuzzy_match_enabled: normalize_config.enabled && normalize_config.fuzzy_matching, - smart_match_config: normalize_config, - - } - } - - pub fn is_empty(&self) -> bool { - self.channel.is_empty() && self.normalized.is_empty() - } - - pub fn normalize_and_store(&mut self, name: &str) { - let normalized_name = self.normalize(name); - let normalized_key: Cow = Cow::Owned(normalized_name.clone()); - - let phonetic_code = if self.smart_match_config.fuzzy_matching { - let code: Cow = Cow::Owned(self.metaphone.encode(&normalized_name)); - self.normalized_phonetic.insert(normalized_key.clone(), code.clone()); - Some(code) - } else { - None - }; - - self.normalized.insert(normalized_key, (phonetic_code, None)); - } - - pub fn normalize(&self, name: &str) -> String { - normalize_channel_name(name, &self.smart_match_config) - } - - pub fn phonetic(&self, name: &str) -> Cow { - self.normalized_phonetic.get(&Cow::Owned(name.to_string())).map_or_else(|| Cow::Owned(self.metaphone.encode(name)), std::clone::Clone::clone) - // match self.normalized_phonetic.entry(name.clone()) { - // Entry::Occupied(entry) => { - // entry.get().clone() - // } - // Entry::Vacant(entry) => { - // - // let code: Cow = Cow::Owned(self.metaphone.encode(&name)); - // // entry.insert(code.clone()); - // code - // } - // } - } -} - -fn prepare_epg_id_cache(fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache) { - let smart_match_enabled = id_cache.smart_match_enabled; - let fuzzy_matching = id_cache.fuzzy_match_enabled; - - for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { - let epg_id = channel.header.epg_channel_id.as_deref(); - - // insert epg_id to known channel epg_ids - if let Some(id) = epg_id { - if !id.is_empty() { - id_cache.channel.insert(Cow::Owned(id.to_string())); - } - } - - // for fuzzy_matching we need to put the normalized name even if there is an epg_id, because the epg_id - // could not match to the epg file. And then we try to guess it based on normalized name - let needs_normalization = smart_match_enabled && (fuzzy_matching || epg_id.is_none_or(str::is_empty)); - - if needs_normalization { - let name = &channel.header.name; - id_cache.normalize_and_store(name); - } - } -} - -fn assign_channel_epg(new_epg: &mut Vec, fp: &mut FetchedPlaylist, id_cache: &mut EpgIdCache) { - - if let Some(tv_guide) = &fp.epg { - if let Some(epg) = tv_guide.filter(id_cache) { - // icon tags - let icon_tags : HashMap<&String, &XmlTag> = epg.children.iter() - .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) - .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t)).collect(); - - let filter_live = |c: &&mut PlaylistItem| c.header.xtream_cluster == XtreamCluster::Live; - let filter_missing_epg_id = |chan: &&mut PlaylistItem | chan.header.epg_channel_id.is_none() || chan.header.logo.is_empty() || chan.header.logo_small.is_empty(); - - let assign_values = |chan: &mut PlaylistItem| { - if id_cache.smart_match_enabled { - // if the channel has no epg_id or the epg_id is not present in xmltv/tvguide then we need to match one from existing tvguide - if chan.header.epg_channel_id.is_none() || !id_cache.processed.contains(chan.header.epg_channel_id.as_ref().unwrap()) { - let normalized = id_cache.normalize(&chan.header.name); - if let Some((_, Some(epg_id))) = id_cache.normalized.get(&Cow::Borrowed(normalized.as_str())) { - chan.header.epg_channel_id = Some(epg_id.to_string()); - } - } - } - if chan.header.epg_channel_id.is_some() && (chan.header.logo.is_empty() || chan.header.logo_small.is_empty()) { - if let Some(icon_tag) = icon_tags.get(chan.header.epg_channel_id.as_ref().unwrap()) { - if let Some(icon) = icon_tag.icon.as_ref() { - if chan.header.logo.is_empty() { - chan.header.logo = (*icon).to_string(); - } - if chan.header.logo_small.is_empty() { - chan.header.logo = (*icon).to_string(); - } - } - } - } - }; - - fp.playlistgroups.iter_mut() - .flat_map(|g| &mut g.channels) - .filter(filter_live) - .filter(filter_missing_epg_id) - .for_each(assign_values); - new_epg.push(epg); - } - } -} - async fn process_playlist_for_target(client: Arc, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, @@ -659,6 +521,10 @@ async fn process_playlist_for_target(client: Arc, let mut duplicates: HashSet = HashSet::new(); let mut processed_fetched_playlists: Vec = vec![]; + + debug!("Executing processing pipes"); + + let mut step = StepMeasure::new("Pipes processed"); for provider_fpl in playlists.iter_mut() { let mut processed_fpl = execute_pipe(target, &pipe, provider_fpl, &mut duplicates); playlist_resolve_series(Arc::clone(&client), cfg, target, errors, &pipe, provider_fpl, &mut processed_fpl).await; @@ -674,40 +540,42 @@ async fn process_playlist_for_target(client: Arc, processed_fetched_playlists.push(processed_fpl); } + step.tick("Processed affixes"); apply_affixes(&mut processed_fetched_playlists); + step.tick("Processed epg"); let (new_epg, new_playlist) = process_epg(&mut processed_fetched_playlists); if new_playlist.is_empty() { info!("Playlist is empty: {}", &target.name); Ok(()) } else { + step.tick("Merged playlists"); let mut flat_new_playlist = flatten_groups(new_playlist); + step.tick("Sorted playlists"); sort_playlist(target, &mut flat_new_playlist); + step.tick("Assigned channel number"); channel_no_playlist(&mut flat_new_playlist); + step.tick("Assigned channel counter"); map_playlist_counter(target, &mut flat_new_playlist); + step.tick("Processed group watches"); process_watch(target, cfg, &flat_new_playlist); - persist_playlist(&mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, cfg).await + step.tick("Persisting playlists"); + let result = persist_playlist(&mut flat_new_playlist, flatten_tvguide(&new_epg).as_ref(), target, cfg).await; + step.tick(""); + result } } fn process_epg(processed_fetched_playlists: &mut Vec) -> (Vec, Vec) { + let mut new_playlist = vec![]; let mut new_epg = vec![]; // each fetched playlist can have its own epgl url. // we need to process each input epg. for fp in processed_fetched_playlists { - // collect all epg_channel ids - let mut id_cache = EpgIdCache::new(fp.input.epg.as_ref()); - prepare_epg_id_cache(fp, &mut id_cache); - - if id_cache.is_empty() { - debug!("channel ids are empty"); - } else { - //debug_if_enabled!("found epg information for {}", &target.name); - assign_channel_epg(&mut new_epg, fp, &mut id_cache); - } + process_playlist_epg(fp, &mut new_epg); new_playlist.append(&mut fp.playlistgroups); } (new_epg, new_playlist) diff --git a/src/utils/mod.rs b/src/utils/mod.rs index b160b213c..7a2e22510 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -11,6 +11,7 @@ pub mod bincode_utils; pub mod time_utils; pub mod crypto_utils; pub mod constants; +pub mod step_measure; #[macro_export] macro_rules! debug_if_enabled { @@ -43,4 +44,4 @@ macro_rules! trace_if_enabled { } pub use debug_if_enabled; -pub use trace_if_enabled; \ No newline at end of file +pub use trace_if_enabled; diff --git a/src/utils/step_measure.rs b/src/utils/step_measure.rs new file mode 100644 index 000000000..aafc3e1a1 --- /dev/null +++ b/src/utils/step_measure.rs @@ -0,0 +1,42 @@ +use std::time::{Duration, Instant}; +use log::{debug, log_enabled, Level}; + +fn format_duration(duration: Duration) -> String { + let millis = duration.as_millis(); + let secs = duration.as_secs(); + let mins = secs / 60; + let secs_rem = secs % 60; + let millis_rem = duration.subsec_millis(); + + if millis < 1_000 { + format!("{} ms", millis) + } else if secs < 60 { + format!("{}.{:03} s", secs, millis_rem) + } else { + format!("{}:{:02}.{:03} min", mins, secs_rem, millis_rem) + } +} + +pub struct StepMeasure { + enabled: bool, + msg: String, + start: Instant, +} + +impl StepMeasure { + pub fn new(msg: &str) -> Self { + Self { + enabled: log_enabled!(Level::Debug), + msg: msg.to_string(), + start: Instant::now(), + } + } + + pub fn tick(&mut self, msg: &str) { + if self.enabled { + debug!("{} in {}", self.msg, format_duration(self.start.elapsed())); + self.msg = msg.to_string(); + self.start = Instant::now(); + } + } +} \ No newline at end of file