added virtual id handling

src/processing/m3u_parser.rs

@@ -4,7 +4,6 @@ use std::rc::Rc;
 
 use crate::model::config::{Config, ConfigInput};
 use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
-use crate::utils::default_utils::{default_as_empty_rc_str, default_playlist_item_type, default_stream_cluster};
 use crate::utils::string_utils;
 
 #[inline]
@@ -76,27 +75,10 @@ fn skip_digit(it: &mut std::str::Chars) -> Option<char> {
 
 fn create_empty_playlistitem_header(input_id: u16, url: &str) -> PlaylistItemHeader {
     PlaylistItemHeader {
-        uuid: default_as_empty_rc_str(),
-        id: default_as_empty_rc_str(),
-        stream_id: default_as_empty_rc_str(),
-        name: default_as_empty_rc_str(),
-        chno: default_as_empty_rc_str(),
-        logo: default_as_empty_rc_str(),
-        logo_small: default_as_empty_rc_str(),
-        group: default_as_empty_rc_str(),
-        title: default_as_empty_rc_str(),
-        parent_code: default_as_empty_rc_str(),
-        audio_track: default_as_empty_rc_str(),
-        time_shift: default_as_empty_rc_str(),
-        rec: default_as_empty_rc_str(),
         url: Rc::new(url.to_owned()),
-        epg_channel_id: None,
-        item_type: default_playlist_item_type(),
-        xtream_cluster: default_stream_cluster(),
-        additional_properties: None,
-        series_fetched: false,
         category_id: 0,
         input_id,
+        ..Default::default()
     }
 }
 
@@ -147,7 +129,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str
                 plih.id = Rc::new(chanid);
             }
         }
-        plih.stream_id = Rc::clone(&plih.id);
+        // plih.virtual_id = plih.id;
         plih.epg_channel_id = Some(Rc::clone(&plih.id));
     }
 
@@ -174,7 +156,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str
 
 pub(crate) fn extract_id_from_url(url: &str) -> Option<String> {
     if let Some(filename) = url.split('/').last() {
-       return if let Some(index) = filename.rfind('.') {
+        return if let Some(index) = filename.rfind('.') {
             Some(filename[..index].to_string())
         } else {
             Some(filename.to_string())
@@ -183,14 +165,17 @@ pub(crate) fn extract_id_from_url(url: &str) -> Option<String> {
     None
 }
 
-pub(crate) fn consume_m3u<F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: impl Iterator<Item=String>, mut visit: F) {
+pub(crate) fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F)
+where
+    I: Iterator<Item=&'a str>,
+{
     let mut header: Option<String> = None;
     let mut group: Option<String> = None;
 
     let video_suffixes = cfg.video.as_ref().unwrap().extensions.iter().map(String::as_str).collect::<Vec<&str>>();
     for line in lines {
         if line.starts_with("#EXTINF") {
-            header = Some(line);
+            header = Some(String::from(line));
             continue;
         }
         if line.starts_with("#EXTGRP") {
@@ -219,29 +204,33 @@ pub(crate) fn consume_m3u<F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigIn
     }
 }
 
-pub(crate) fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: &[String]) -> Vec<PlaylistGroup> {
+pub(crate) fn parse_m3u<'a, I>(cfg: &Config, input: &ConfigInput, lines: I) -> Vec<PlaylistGroup>
+where
+    I: Iterator<Item=&'a str>,
+{
     let mut sort_order: Vec<Vec<PlaylistItem>> = vec![];
-    let mut idx: usize = 0;
+    let mut sort_order_idx: usize = 0;
     let mut group_map: std::collections::HashMap<Rc<String>, usize> = std::collections::HashMap::new();
-    consume_m3u(cfg, input, lines.iter().cloned(), |item| {
+    consume_m3u(cfg, input, lines, |item| {
+        // keep the original sort order for groups and group the playlist items
         let key = Rc::clone(&item.header.borrow().group);
         match group_map.entry(key) {
             std::collections::hash_map::Entry::Vacant(v) => {
-                v.insert(idx);
-                idx += 1;
+                v.insert(sort_order_idx);
                 sort_order.push(vec![item]);
+                sort_order_idx += 1;
             }
             std::collections::hash_map::Entry::Occupied(o) => {
                 sort_order.get_mut(*o.get()).unwrap().push(item);
             }
         }
     });
-
     let mut grp_id = 0;
     let result: Vec<PlaylistGroup> = sort_order.drain(..).map(|channels| {
+        // create a group based on the first playlist item
         let channel = channels.first();
-        let cluster = channel.map(|pli| pli.header.borrow().xtream_cluster).unwrap();
-        let group_title = channel.map(|pli| Rc::clone(&pli.header.borrow().group)).unwrap();
+        let (cluster, group_title) = channel.map(|pli|
+                                                 (pli.header.borrow().xtream_cluster, Rc::clone(&pli.header.borrow().group))).unwrap();
         grp_id += 1;
         PlaylistGroup { id: grp_id, xtream_cluster: cluster, title: Rc::clone(&group_title), channels }
     }).collect();

src/processing/playlist_processor.rs

@@ -25,8 +25,8 @@ use crate::processing::playlist_watch::process_group_watch;
 use crate::processing::xmltv_parser::flatten_tvguide;
 use crate::processing::xtream_processor::playlist_resolve_series;
 use crate::repository::playlist_repository::persist_playlist;
-use crate::utils::download;
 use crate::utils::default_utils::default_as_default;
+use crate::utils::download;
 
 fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool {
     let provider = ValueProvider { pli: RefCell::new(pli) };
@@ -96,7 +96,7 @@ fn playlistitem_comparator(a: &PlaylistItem, b: &PlaylistItem, channel_sort: &Co
                     }
                 }
             }
-        },
+        }
         None => {
             let ordering = value_a.partial_cmp(&value_b).unwrap();
             match channel_sort.order {
@@ -255,7 +255,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) {
                         for channel in &plg.channels {
                             let provider = ValueProvider { pli: RefCell::new(channel) };
                             if counter.filter.filter(&provider, &mut mock_processor) {
-                               let new_value = if counter.modifier == CounterModifier::Assign {
+                                let new_value = if counter.modifier == CounterModifier::Assign {
                                     cntval.to_string()
                                 } else {
                                     let value = match channel.header.borrow_mut().get_field(&counter.field) {
@@ -264,7 +264,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) {
                                     };
                                     if counter.modifier == CounterModifier::Suffix {
                                         format!("{}{}{}", value, counter.concat, cntval.to_string())
-                                    }  else {
+                                    } else {
                                         format!("{}{}{}", cntval.to_string(), counter.concat, value)
                                     }
                                 };
@@ -295,10 +295,11 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
 
 async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<ProcessTargets>) -> (Vec<InputStats>, Vec<M3uFilterError>) {
     let source = cfg.sources.get(source_idx).unwrap();
-    let mut source_playlists = Vec::new();
-    let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count();
     let mut errors = vec![];
     let mut stats = HashMap::<u16, InputStats>::new();
+    let mut source_playlists = Vec::new();
+    let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count();
+    // Downlod the sources
     for input in &source.inputs {
         let input_id = input.id;
         if is_input_enabled(enabled_inputs, input.enabled, input_id, &user_targets) {
@@ -306,16 +307,16 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
                 InputType::M3u => download::get_m3u_playlist(&cfg, input, &cfg.working_dir).await,
                 InputType::Xtream => download::get_xtream_playlist(input, &cfg.working_dir).await,
             };
+            // @TODO optmization dont hold tv_guide in memory, persist raw and  later use sax parser to extract.
             let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
                 download::get_xmltv(&cfg, input, &cfg.working_dir).await
             } else {
                 (None, vec![])
             };
-            error_list.drain(..).for_each(|err| errors.push(err));
-            tvguide_errors.drain(..).for_each(|err| errors.push(err));
+            errors.extend(error_list.drain(..).chain(tvguide_errors.drain(..)));
             let input_name = match &input.name {
+                Some(name_val) => name_val.as_str(),
                 None => input.url.as_str(),
-                Some(name_val) => name_val.as_str()
             };
             let group_count = playlistgroups.len();
             let channel_count = playlistgroups.iter()
@@ -334,19 +335,8 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
                     }
                 );
             }
-            stats.insert(input_id, InputStats {
-                name: input_name.to_string(),
-                input_type: input.input_type.clone(),
-                error_count: error_list.len(),
-                raw_stats: PlaylistStats {
-                    group_count,
-                    channel_count,
-                },
-                processed_stats: PlaylistStats {
-                    group_count: 0,
-                    channel_count: 0,
-                },
-            });
+            stats.insert(input_id, create_input_stat(group_count, channel_count, error_list.len(),
+                                                     input.input_type.clone(), input_name));
         }
     }
     if source_playlists.is_empty() {
@@ -362,7 +352,7 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
             if is_target_enabled(target, &user_targets) {
                 match process_playlist(&mut source_playlists, target, &cfg, &mut stats, &mut errors).await {
                     Ok(()) => {}
-                    Err(mut err) => err.drain(..).for_each(|e| errors.push(e))
+                    Err(mut err) => errors.extend(err.drain(..))
                 }
             }
         }
@@ -370,6 +360,22 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
     (stats.drain().map(|(_, v)| v).collect(), errors)
 }
 
+fn create_input_stat(group_count: usize, channel_count: usize, error_count: usize, input_type: InputType, input_name: &str) -> InputStats {
+    InputStats {
+        name: input_name.to_string(),
+        input_type: input_type,
+        error_count: error_count,
+        raw_stats: PlaylistStats {
+            group_count,
+            channel_count,
+        },
+        processed_stats: PlaylistStats {
+            group_count: 0,
+            channel_count: 0,
+        },
+    }
+}
+
 async fn process_sources(config: Arc<Config>, user_targets: Arc<ProcessTargets>) -> (Vec<InputStats>, Vec<M3uFilterError>) {
     let mut handle_list = vec![];
     let thread_num = config.threads;
@@ -390,10 +396,8 @@ async fn process_sources(config: Arc<Config>, user_targets: Arc<ProcessTargets>)
                 let (mut res_stats, mut res_errors) = System::new().block_on(async {
                     process_source(cfg, index, usr_trgts).await
                 });
-                res_errors.drain(..)
-                    .for_each(|err| shared_errors.lock().unwrap().push(err));
-                res_stats.drain(..)
-                    .for_each(|stat| shared_stats.lock().unwrap().push(stat));
+                shared_errors.lock().unwrap().extend(res_errors.drain(..));
+                shared_stats.lock().unwrap().extend(res_stats.drain(..));
             };
             handles.push(thread::spawn(process));
             if handles.len() >= thread_num as usize {
@@ -401,10 +405,8 @@ async fn process_sources(config: Arc<Config>, user_targets: Arc<ProcessTargets>)
             }
         } else {
             let (mut res_stats, mut res_errors) = process_source(cfg, index, usr_trgts).await;
-            res_errors.drain(..)
-                .for_each(|err| shared_errors.lock().unwrap().push(err));
-            res_stats.drain(..)
-                .for_each(|stat| shared_stats.lock().unwrap().push(stat));
+            shared_errors.lock().unwrap().extend(res_errors.drain(..));
+            shared_stats.lock().unwrap().extend(res_stats.drain(..));
         }
     }
     for handle in handle_list {
@@ -493,12 +495,11 @@ async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>],
     let mut new_epg = vec![];
 
     new_fetched_playlists.drain(..).for_each(|mut fp| {
+        // collect all epg_channel ids
         let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels)
             .filter_map(|c| c.header.borrow().epg_channel_id.clone()).collect();
 
-        fp.playlistgroups.drain(..).for_each(|group| {
-            new_playlist.push(group);
-        });
+        new_playlist.extend(fp.playlistgroups.drain(..));
         if !epg_channel_ids.is_empty() {
             if let Some(tv_guide) = fp.epg {
                 debug!("found epg information for {}", &target.name);

src/processing/xtream_parser.rs

@@ -8,12 +8,11 @@ use crate::create_m3u_filter_error_result;
 use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
 use crate::model::config::ConfigInput;
 use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
-use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamStream};
-use crate::utils::default_utils::default_as_empty_rc_str;
+use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream};
 
-fn map_to_xtream_category(category: &Value) -> Result<Vec<XtreamCategory>, M3uFilterError> {
-    match serde_json::from_value::<Vec<XtreamCategory>>(category.to_owned()) {
-        Ok(category_list) => Ok(category_list),
+fn map_to_xtream_category(categories: &Value) -> Result<Vec<XtreamCategory>, M3uFilterError> {
+    match serde_json::from_value::<Vec<XtreamCategory>>(categories.to_owned()) {
+        Ok(xtream_categories) => Ok(xtream_categories),
         Err(err) => {
             create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "Failed to process categories {}", &err)
         }
@@ -29,6 +28,16 @@ fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: &Value) -> Resu
     }
 }
 
+fn create_xtream_series_info_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Rc<String> {
+    if episode.direct_source.is_empty() {
+        let ext = episode.container_extension.clone();
+        let stream_base_url = format!("{url}/series/{username}/{password}/{}.{ext}", episode.id);
+        Rc::new(stream_base_url)
+    } else {
+        Rc::new(episode.direct_source.clone())
+    }
+}
+
 pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &ConfigInput) -> Result<Option<Vec<PlaylistItem>>, M3uFilterError> {
     let url = input.url.as_str();
     let username = input.username.as_ref().map_or("", |v| v);
@@ -39,33 +48,18 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &
             let result: Vec<PlaylistItem> = series_info.episodes.values().flatten().map(|episode|
                 PlaylistItem {
                     header: RefCell::new(PlaylistItemHeader {
-                        uuid: default_as_empty_rc_str(),
-                        id: Rc::new(episode.id.clone()),
-                        stream_id: Rc::new(episode.id.clone()),
+                        id: Rc::new(episode.id.to_string()),
                         name: Rc::new(episode.title.clone()),
-                        chno: default_as_empty_rc_str(),
                         logo: Rc::new(episode.info.movie_image.clone()),
-                        logo_small: default_as_empty_rc_str(),
                         group: Rc::new(group_title.to_string()),
                         title: Rc::new(episode.title.clone()),
-                        parent_code: default_as_empty_rc_str(),
-                        audio_track: default_as_empty_rc_str(),
-                        time_shift: default_as_empty_rc_str(),
-                        rec: default_as_empty_rc_str(),
-                        url: if episode.direct_source.is_empty() {
-                            let ext = episode.container_extension.clone();
-                            let stream_base_url = format!("{}/series/{}/{}/{}.{}", url, username, password, episode.id.as_str(), ext);
-                            Rc::new(stream_base_url)
-                        } else {
-                            Rc::new(episode.direct_source.clone())
-                        },
-                        epg_channel_id: None,
+                        url: create_xtream_series_info_url(url, username, password, episode),
                         item_type: PlaylistItemType::Series,
                         xtream_cluster: XtreamCluster::Series,
                         additional_properties: episode.get_additional_properties(&series_info),
-                        series_fetched: false,
                         category_id: 0,
                         input_id: input.id,
+                        ..Default::default()
                     })
                 }).collect();
             if result.is_empty() { Ok(None) } else { Ok(Some(result)) }
@@ -76,69 +70,61 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &
     }
 }
 
+fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &str, password: &str, stream: &XtreamStream) -> Rc<String> {
+    if stream.direct_source.is_empty() {
+        let stream_base_url = match xtream_cluster {
+            XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()),
+            XtreamCluster::Video => {
+                let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str());
+                format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext)
+            }
+            XtreamCluster::Series =>
+                format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}",
+                        url, username, password, &stream.get_stream_id())
+        };
+        Rc::new(stream_base_url)
+    } else {
+        Rc::clone(&stream.direct_source)
+    }
+}
 pub(crate) fn parse_xtream(input: &ConfigInput,
                            xtream_cluster: XtreamCluster,
-                           category: &Value,
+                           categories: &Value,
                            streams: &Value) -> Result<Option<Vec<PlaylistGroup>>, M3uFilterError> {
-    match map_to_xtream_category(category) {
-        Ok(mut categories) => {
+    match map_to_xtream_category(categories) {
+        Ok(mut xtream_categories) => {
             let input_id = input.id;
             let url = input.url.as_str();
             let username = input.username.as_ref().map_or("", |v| v);
             let password = input.password.as_ref().map_or("", |v| v);
 
             return match map_to_xtream_streams(xtream_cluster, streams) {
-                Ok(streams) => {
+                Ok(xtream_streams) => {
                     let group_map: HashMap::<Rc<String>, RefCell<XtreamCategory>> =
-                        categories.drain(..).map(|category|
+                        xtream_categories.drain(..).map(|category|
                             (Rc::clone(&category.category_id), RefCell::new(category))
                         ).collect();
 
-                    for stream in streams {
+                    for stream in xtream_streams {
                         if let Some(group) = group_map.get(&stream.category_id) {
                             let mut grp = group.borrow_mut();
                             let category_name = &grp.category_name;
                             let item = PlaylistItem {
                                 header: RefCell::new(PlaylistItemHeader {
-                                    uuid: default_as_empty_rc_str(),
-                                    id: Rc::new(stream.get_stream_id()),
-                                    stream_id: Rc::new(stream.get_stream_id()),
+                                    id: Rc::new(stream.get_stream_id().to_string()),
                                     name: Rc::clone(&stream.name),
-                                    chno: default_as_empty_rc_str(),
                                     logo: Rc::clone(&stream.stream_icon),
-                                    logo_small: default_as_empty_rc_str(),
                                     group: Rc::clone(category_name),
                                     title: Rc::clone(&stream.name),
-                                    parent_code: default_as_empty_rc_str(),
-                                    audio_track: default_as_empty_rc_str(),
-                                    time_shift: default_as_empty_rc_str(),
-                                    rec: default_as_empty_rc_str(),
-                                    url: if stream.direct_source.is_empty() {
-                                        let stream_base_url = match xtream_cluster {
-                                            XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()),
-                                            XtreamCluster::Video => {
-                                                let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str());
-                                                format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext)
-                                            }
-                                            XtreamCluster::Series =>
-                                                format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}",
-                                                        url, username, password, &stream.get_stream_id())
-                                        };
-                                        Rc::new(stream_base_url)
-                                    } else {
-                                        Rc::clone(&stream.direct_source)
-                                    },
+                                    url: create_xtream_url(xtream_cluster, url, username, password, &stream),
                                     epg_channel_id: stream.epg_channel_id.clone(),
-                                    item_type: match xtream_cluster {
-                                        XtreamCluster::Live => PlaylistItemType::Live,
-                                        XtreamCluster::Video => PlaylistItemType::Movie,
-                                        XtreamCluster::Series => PlaylistItemType::SeriesInfo,
-                                    },
+                                    item_type: PlaylistItemType::from(xtream_cluster),
                                     xtream_cluster,
                                     additional_properties: stream.get_additional_properties(),
                                     series_fetched: false,
                                     category_id: 0,
                                     input_id,
+                                    ..Default::default()
                                 }),
                             };
                             grp.add(item);
@@ -160,4 +146,5 @@ pub(crate) fn parse_xtream(input: &ConfigInput,
         }
         Err(err) => Err(err)
     }
-}
+}
+