xtream support

README.md

@@ -74,16 +74,40 @@ Has five entries: `enabled`, `persist`, `url`, `prefix`, `suffix`.
   - `enabled` is optional, default is true, if you disable the processing is skipped 
   - `persist` is optional, you can skip or leave it blank to avoid persisting the input file. The `{}` in the filename is filled with the current timestamp.
   - `url` for type `m3u` is the download url or a local filename of the input-source. For type `xtream`it is `http://<hostname>:<port>`
+  - `headers` is optional, used only for type `xtream`
   - `username` only mandatory for type `xtream`
   - `pasword`only mandatory for type `xtream`
   - `prefix` is optional, it is applied to the given field with the given value
   - `suffix` is optional, it is applied to the given field with the given value
 
-`prefix` and `suffix` is appended after all processing is done, but before sort.
+`prefix` and `suffix` are appended after all processing is done, but before sort.
 They have 2 fields:
   - `field` can be `name` , `group`, `title`
   - `value` a static text
 
+Example input config for `m3u`
+```
+sources:
+  - input:
+      persist: 'playlist_1_{}.m3u'
+      url: 'http://localhost:8080/get.php?username=test&password=test&type=m3u'
+```
+
+Example input config for `xtream`
+```
+sources:
+  - input:
+      type: xtream
+      headers:
+        User-Agent: "Mozilla/5.0 (Linux; Tizen 2.3) AppleWebKit/538.1 (KHTML, like Gecko)Version/2.3 TV Safari/538.1"
+        Accept: application/json
+        Accept-Encoding: gzip
+      url: 'http://localhost:8080'
+      username: test
+      password: test
+```
+
+
 ### 1.5.2 `targets`
 Has the following top level entries:
 * `enabled` _optional_ default is `true`, if you disable the processing is skipped

config.yml

@@ -1,6 +1,18 @@
 api: { host: 127.0.0.1, port: 8901, web_root: /home/m3u-filter/web }
 threads: 0
 working_dir: /home/m3u-filter/data
+templates:
+  - name: DE_CHAN
+    value: >-
+      Group ~ "(?i)^.DE.*Serien.*" OR
+      Group ~ "(?i)^.TR.*Filme.*" OR
+      Group ~ "(?i)^.TR.*Live.*"
+  - name: FR_CHAN
+    value: Group ~ "^.FR..*"
+  - name: ALL_CHAN
+    value: >-
+      !DE_CHAN! OR
+      !FR_CHAN!
 sources:
   - input:
       type: xtream
@@ -8,19 +20,14 @@ sources:
         User-Agent: "Mozilla/5.0 (Linux; Tizen 2.3) AppleWebKit/538.1 (KHTML, like Gecko)Version/2.3 TV Safari/538.1"
         Accept: application/json
         Accept-Encoding: gzip
-
-      m3u:
-        persist: 'playlist_1_{}.m3u'
-        url: 'http://localhost/playlist.m3u'
-      xtream:
-        url: 'http://localhost:3000'
-        username: test
-        password: test
+      url: 'http://localhost:8080'
+      username: test
+      password: test
     targets:
       - name: pl1
         filename: playlist_1.m3u
         type: m3u
-        filter: 'Group ~ "^DE\s.*" OR Group ~ "TR\s.*" OR Group ~ "TF1.*"'
+        filter: '!ALL_CHAN!'
         options: { ignore_logo: true }
         rename:
           - { field: group, new_name: 1. DE$1, pattern: ^DE(.*) }

src/m3u_processor.rs

@@ -52,7 +52,7 @@ fn skip_digit(it: &mut std::str::Chars) -> Option<char> {
     }
 }
 
-fn decode_header(content: &String) -> PlaylistItemHeader {
+fn process_header(content: &String) -> PlaylistItemHeader {
     let mut plih = PlaylistItemHeader {
         id: String::from(""),
         name: String::from(""),
@@ -103,7 +103,7 @@ fn decode_header(content: &String) -> PlaylistItemHeader {
     plih
 }
 
-pub(crate) fn decode(lines: &Vec<String>) -> Vec<PlaylistGroup> {
+pub(crate) fn process(lines: &Vec<String>) -> Vec<PlaylistGroup> {
     let mut groups: std::collections::HashMap<String, Vec<PlaylistItem>> = std::collections::HashMap::new();
     let mut sort_order: Vec<String> = vec![];
     let mut header: Option<String> = None;
@@ -122,7 +122,7 @@ pub(crate) fn decode(lines: &Vec<String>) -> Vec<PlaylistGroup> {
             continue;
         }
         if header.is_some() {
-            let mut item = PlaylistItem { header: decode_header(&header.unwrap()), url: String::from(line) };
+            let mut item = PlaylistItem { header: process_header(&header.unwrap()), url: String::from(line) };
             if group.is_some() && item.header.group.is_empty() {
                 item.header.group = String::from(group.unwrap());
             }

src/service.rs

@@ -22,7 +22,7 @@ pub(crate) fn get_m3u_playlist(input: &ConfigInput, working_dir: &String, verbos
     let url = input.url.as_str();
     let file_path = prepare_file_path(input, working_dir, "", verbose);
     let lines: Option<Vec<String>> = utils::get_input_content(working_dir, url, file_path, verbose);
-    lines.map_or(None, |l| Some(m3u_processor::decode(&l)))
+    lines.map_or(None, |l| Some(m3u_processor::process(&l)))
 }
 
 
@@ -41,9 +41,9 @@ pub(crate) fn get_xtream_playlist(input: &ConfigInput, working_dir: &String, ver
         let category_file_path = prepare_file_path(input, working_dir, format!("{}_", category).as_str(), verbose);
         let stream_file_path = prepare_file_path(input, working_dir, format!("{}_", stream).as_str(), verbose);
 
-        let category_content: Option<serde_json::Value> = utils::get_input_json_content(input, working_dir, &category_url, category_file_path, verbose);
-        let stream_content: Option<serde_json::Value> = utils::get_input_json_content(input, working_dir, &stream_url, stream_file_path, verbose);
-        let mut sub_playlist = xtream_processor::decode(category_content, stream_content, &stream_base_url);
+        let category_content: Option<serde_json::Value> = utils::get_input_json_content(input, &category_url, category_file_path, verbose);
+        let stream_content: Option<serde_json::Value> = utils::get_input_json_content(input, &stream_url, stream_file_path, verbose);
+        let mut sub_playlist = xtream_processor::process(category_content, stream_content, &stream_base_url);
         while let Some(group) = sub_playlist.pop() {
             playlist.push(group);
         }

src/utils.rs

@@ -188,7 +188,7 @@ pub(crate) fn get_file_path(wd: &String, path: Option<PathBuf>) -> Option<PathBu
     }
 }
 
-fn download_json_content(input: &ConfigInput, url: url::Url, _persist_file: Option<PathBuf>, verbose: bool) -> Result<serde_json::Value, String> {
+fn download_json_content(input: &ConfigInput, url: url::Url, persist_file: Option<PathBuf>, verbose: bool) -> Result<serde_json::Value, String> {
     let mut request = reqwest::blocking::Client::new().get(url);
     if input.headers.is_empty() {
         let mut headers = header::HeaderMap::new();
@@ -206,7 +206,7 @@ fn download_json_content(input: &ConfigInput, url: url::Url, _persist_file: Opti
             if response.status().is_success() {
                 match response.json::<serde_json::Value>() {
                     Ok(content) => {
-                        //persist_playlist(persist_file, , verbose);
+                        persist_playlist(persist_file, &serde_json::to_string(&content).unwrap(), verbose);
                         Ok(content)
                     }
                     Err(e) => Err(e.to_string())
@@ -219,7 +219,7 @@ fn download_json_content(input: &ConfigInput, url: url::Url, _persist_file: Opti
     }
 }
 
-pub(crate) fn get_input_json_content(input: &ConfigInput, _working_dir: &String, url_str: &String, persist_file: Option<PathBuf>, verbose: bool) -> Option<serde_json::Value> {
+pub(crate) fn get_input_json_content(input: &ConfigInput, url_str: &String, persist_file: Option<PathBuf>, verbose: bool) -> Option<serde_json::Value> {
     match url_str.parse::<url::Url>() {
         Ok(url) => match download_json_content(input, url, persist_file, verbose) {
             Ok(content) => Some(content),

src/xtream_processor.rs

@@ -1,15 +1,27 @@
 use std::cell::RefCell;
 use std::collections::HashMap;
-use serde::{Deserialize};
+use serde::{Deserialize, Deserializer};
 use crate::m3u::{PlaylistGroup, PlaylistItem, PlaylistItemHeader};
 use crate::model::{default_as_empty_str};
 
+fn null_to_default<'de, D, T>(d: D) -> Result<T, D::Error>
+    where
+        D: Deserializer<'de>,
+        T: Default + Deserialize<'de>,
+{
+    let opt = Option::deserialize(d)?;
+    let val = opt.unwrap_or_else(T::default);
+    Ok(val)
+}
 
 pub fn default_as_empty_list() -> Vec<PlaylistItem> { vec![] }
+pub fn default_as_zero() -> i32 { 0 }
 
 #[derive(Deserialize)]
 struct XtreamCategory {
+    #[serde(deserialize_with = "null_to_default")]
     pub category_id: String,
+    #[serde(deserialize_with = "null_to_default")]
     pub category_name: String,
     //pub parent_id: i32,
     #[serde(default = "default_as_empty_list")]
@@ -24,22 +36,28 @@ impl XtreamCategory {
 
 #[derive(Deserialize)]
 struct XtreamStream {
+    #[serde(default = "default_as_zero", deserialize_with = "null_to_default")]
     pub num: i32,
+    #[serde(deserialize_with = "null_to_default")]
     pub name: String,
     //pub stream_type: String,
+    #[serde(deserialize_with = "null_to_default")]
+    pub category_id: String,
+
+    #[serde(default = "default_as_zero", deserialize_with = "null_to_default")]
     pub stream_id: i32,
+    #[serde(default = "default_as_empty_str", deserialize_with = "null_to_default")]
     pub stream_icon: String,
     //pub epg_channel_id: String,
     // pub added: String,
-    pub category_id: String,
     //pub custom_sid: String,
     //pub tv_archive: i32,
-    #[serde(default = "default_as_empty_str")]
+    #[serde(default = "default_as_empty_str", deserialize_with = "null_to_default")]
     pub direct_source: String,
     //pub tv_archive_duration: i32,
 }
 
-fn decode_category(category: Option<serde_json::Value>) -> Vec<XtreamCategory> {
+fn process_category(category: Option<serde_json::Value>) -> Vec<XtreamCategory> {
     match category {
         Some(value) => {
             match serde_json::from_value::<Vec<XtreamCategory>>(value) {
@@ -55,7 +73,7 @@ fn decode_category(category: Option<serde_json::Value>) -> Vec<XtreamCategory> {
 }
 
 
-fn decode_streams(streams: Option<serde_json::Value>) -> Vec<XtreamStream> {
+fn process_streams(streams: Option<serde_json::Value>) -> Vec<XtreamStream> {
     match streams {
         Some(value) => {
             match serde_json::from_value::<Vec<XtreamStream>>(value) {
@@ -70,10 +88,10 @@ fn decode_streams(streams: Option<serde_json::Value>) -> Vec<XtreamStream> {
     }
 }
 
-pub(crate) fn decode(category: Option<serde_json::Value>, streams: Option<serde_json::Value>, stream_base_url: &String) -> Vec<PlaylistGroup> {
-    let mut categories = decode_category(category);
+pub(crate) fn process(category: Option<serde_json::Value>, streams: Option<serde_json::Value>, stream_base_url: &String) -> Vec<PlaylistGroup> {
+    let mut categories = process_category(category);
     if !categories.is_empty() {
-        let streams = decode_streams(streams);
+        let streams = process_streams(streams);
         if !streams.is_empty() {
             let mut group_map = HashMap::<String, RefCell<XtreamCategory>>::new();
             while let Some(category) = categories.pop() {