From 3c6977f7227a40594dc1e921de85d4f416aa1baf Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 20 Oct 2023 13:09:20 +0200 Subject: [PATCH] Added multiple target output formats --- CHANGELOG.md | 1 + README.md | 51 +++++++++++---------- src/api/m3u_api.rs | 3 +- src/model/config.rs | 67 +++++++++++++++++++++------- src/model/model_config.rs | 2 +- src/processing/playlist_processor.rs | 37 ++++----------- src/repository/m3u_repository.rs | 16 +++---- src/utils.rs | 10 ++--- 8 files changed, 105 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 561711957..4824c68bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * Added Target watch for groups * Fixed TLS problem with docker scratch * Added simple stats +* Target Output is now a list of multiple output formats, !breaking change! Changes in `config.yml` ```yaml diff --git a/README.md b/README.md index 2ad63a9fe..ca5d91c84 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,9 @@ sources: - url: 'https://raw.githubusercontent.com/iptv-org/iptv/master/streams/za.m3u' targets: - name: test - filename: test.m3u + output: + - type: m3u + filename: test.m3u ``` @@ -155,9 +157,8 @@ sources: Has the following top level entries: * `enabled` _optional_ default is `true`, if you disable the processing is skipped * `name` _optional_ default is `default`, if not default it has to be unique, for running selective targets -* `filename` _mandatory_ if type `m3u` or `strm`, otherwise _optional_ * `sort` _optional_ -* `output` _optional_ list of output types [`m3u`, `strm`, `xtream`], default is [`m3u`] +* `output` _optional_ list of output formats * `processing_order` _optional_ default is `frm` * `options` _optional_ * `filter` _mandatory_, @@ -165,12 +166,7 @@ Has the following top level entries: * `mapping` _optional_ * `watch` _optional_ -### 1.5.2.1 `filename` - -`filename` is the file for resulting playlist. -It is _mandatory_ if output type contains `m3u` or `strm` - -### 1.5.2.2 `sort` +### 1.5.2.1 `sort` Has three top level attributes * `match_as_ascii` _optional_ default is `false` * `groups` @@ -194,19 +190,26 @@ sort: - { field: name, group_pattern: '^DE.*', order: asc } ``` -### 1.5.2.3 `output` -Is a list of target types. -There are three types of targets ```m3u```, ```strm``` and ```xtream```. -If the attribute is not specified ```m3u``` is created by default. -You can set options for each `output` type. +### 1.5.2.2 `output` + +Is a list of output format: +Each format has 2 properties `type` and `filename` +`type` _mandatory_ ```m3u```, ```strm``` and ```xtream```. +`filename` _mandatory_ if type `m3u` or `strm`, otherwise ignored `strm` output has additional options `underscore_whitespace`, `cleanup` and `kodi_style`. -### 1.5.2.4 `processing_order` +```yaml +output: + - type: m3u + filename: {}.m3u +``` + +### 1.5.2.3 `processing_order` The processing order (Filter, Rename and Map) can be configured for each target with: `processing_order: frm` (valid values are: frm, fmr, rfm, rmf, mfr, mrf. default is frm) -### 1.5.2.5 `options` +### 1.5.2.4 `options` * ignore_logo `true` or `false` * underscore_whitespace `true` or `false` * cleanup `true` or `false` @@ -219,7 +222,7 @@ The processing order (Filter, Rename and Map) can be configured for each target - `cleanup` deletes the directory given at `filename`. - `kodi_style` tries to rename `filename` with [kodi style](https://kodi.wiki/view/Naming_video_files/TV_shows). -### 1.5.2.6 `filter` +### 1.5.2.5 `filter` The filter is a string with a filter statement. The filter can have UnaryExpression `NOT`, BinaryExpression `AND OR`, and Comparison `(Group|Title|Name|Url) ~ "regexp"`. Filter fields are `Group`, `Title`, `Name` and `Url`. @@ -230,7 +233,7 @@ If you use characters like `+ | [ ] ( )` in filters don't forget to escape them! The regular expression syntax is similar to Perl-style regular expressions, but lacks a few features like look around and backreferences. -### 1.5.2.7 `rename` +### 1.5.2.6 `rename` Is a List of rename configurations. Each configuration has 3 top level entries. * `field` can be `group`, `title`, `name` or `url`. * `pattern` is a regular expression like `'^TR.:\s?(.*)'` @@ -247,7 +250,7 @@ In the above example each entry starting with `DE` will be prefixed with `1.`. (_Please be aware of the processing order. If you first map, you should match the mapped entries!_) -### 1.5.2.8 `mapping` +### 1.5.2.7 `mapping` `mapping: ` The mappings are defined in a file `mapping.yml`. The filename can be given as `-m` argument. @@ -284,7 +287,9 @@ sources: persist: ./playlist_{}.m3u targets: - name: pl1 - filename: playlist_1.m3u + output: + - type: m3u + filename: playlist_1.m3u processing_order: frm options: ignore_logo: true @@ -297,9 +302,9 @@ sources: new_name: 1. DE$1 - name: pl1strm enabled: false - filename: playlist_strm output: - - strm + - type: strm + filename: playlist_strm options: ignore_logo: true underscore_whitespace: false @@ -316,7 +321,7 @@ sources: new_name: 1. DE$1 ``` -### 1.5.2.9 `watch` +### 1.5.2.8 `watch` For each target with a *unique name*, you can define a watched groups. It is a list of final group names from this target playlist. Final means in this case: the name in the resulting playlist after applying all steps diff --git a/src/api/m3u_api.rs b/src/api/m3u_api.rs index 967e3f3b9..e7038ca16 100644 --- a/src/api/m3u_api.rs +++ b/src/api/m3u_api.rs @@ -18,7 +18,8 @@ pub(crate) async fn m3u_api( ) -> HttpResponse { match _app_state.config.get_target_for_user(api_req.username.as_str(), api_req.password.as_str()) { Some(target) => { - match get_m3u_file_path(&_app_state.config, target) { + let filename = target.get_m3u_filename(); + match get_m3u_file_path(&_app_state.config, &filename) { Some(file_path) => { serve_file(&file_path, &req).await } diff --git a/src/model/config.rs b/src/model/config.rs index 5d92ceca2..01acf3e51 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use enum_iterator::Sequence; -use log::{debug, error}; +use log::{debug, error, warn}; use path_absolutize::*; use crate::filter::{Filter, get_filter, MockValueProcessor, PatternTemplate, prepare_templates, ValueProvider}; @@ -12,7 +12,7 @@ use crate::model::mapping::Mapping; use crate::model::mapping::Mappings; use crate::model::model_config::{default_as_false, default_as_true, default_as_zero, ItemField, ProcessingOrder, SortOrder, TargetType}; use crate::utils; -use crate::utils::{dedup, get_working_path}; +use crate::utils::get_working_path; fn default_as_frm() -> ProcessingOrder { ProcessingOrder::Frm } @@ -148,7 +148,12 @@ pub(crate) struct ConfigOptions { pub kodi_style: bool, } -fn default_as_empty_output() -> Vec:: { vec![] } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct TargetOutput { + #[serde(alias = "type")] + pub target: TargetType, + pub filename: Option, +} #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct ConfigTarget { @@ -158,12 +163,11 @@ pub(crate) struct ConfigTarget { pub enabled: bool, #[serde(default = "default_as_default")] pub name: String, - pub filename: Option, pub options: Option, pub sort: Option, pub filter: String, - #[serde(alias = "type", default = "default_as_empty_output")] - pub output: Vec, + #[serde(alias = "type")] + pub output: Vec, pub rename: Option>, pub mapping: Option>, #[serde(default = "default_as_frm")] @@ -176,9 +180,36 @@ pub(crate) struct ConfigTarget { } impl ConfigTarget { - pub fn prepare(&mut self, id: u16, templates: Option<&Vec>) -> Result<(), M3uFilterError> { + pub(crate) fn prepare(&mut self, id: u16, templates: Option<&Vec>) -> Result<(), M3uFilterError> { self.id = id; - dedup::(&mut self.output); + if self.output.is_empty() { + return Err(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Missing output format for {}", self.name))); + } + for format in &self.output { + match format.target { + TargetType::M3u => { + if format.filename.is_none() { + return create_m3u_filter_error_result!(M3uFilterErrorKind::Info, "filename is required for m3u type: {}", self.name); + } + } + TargetType::Strm => { + if format.filename.is_none() { + return create_m3u_filter_error_result!(M3uFilterErrorKind::Info, "filename is required for strm type: {}", self.name); + } + } + TargetType::Xtream => { + if default_as_default().eq_ignore_ascii_case(&self.name) { + return create_m3u_filter_error_result!(M3uFilterErrorKind::Info, "unique target name is required for xtream type: {}", self.name); + } + if let Some(fname) = &format.filename { + if !fname.trim().is_empty() { + warn!("Filename for target output xtream is ignored: {}", self.name) + } + } + } + } + } + match get_filter(&self.filter, templates) { Ok(fltr) => { debug!("Filter: {}", fltr); @@ -194,10 +225,21 @@ impl ConfigTarget { Err(err) => Err(err), } } - pub fn filter(&self, provider: &ValueProvider) -> bool { + pub(crate) fn filter(&self, provider: &ValueProvider) -> bool { let mut processor = MockValueProcessor {}; return self._filter.as_ref().unwrap().filter(provider, &mut processor); } + + pub(crate) fn get_m3u_filename(&self) -> Option { + for format in &self.output { + match format.target { + TargetType::M3u => return format.filename.clone(), + TargetType::Strm => {} + TargetType::Xtream => {} + } + } + None + } } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -207,7 +249,7 @@ pub(crate) struct ConfigSource { } impl ConfigSource { - pub fn prepare(&mut self, index: u16) -> Result { + pub(crate) fn prepare(&mut self, index: u16) -> Result { handle_m3u_filter_error_result_list!(M3uFilterErrorKind::Info, self.inputs.iter_mut().enumerate().map(|(idx, i)| i.prepare(index+(idx as u16)))); Ok(index + (self.inputs.len() as u16)) } @@ -369,11 +411,6 @@ impl Config { if let Some(mapping_list) = mappings { for source in &mut self.sources { for target in &mut source.targets { - let needs_filename = target.output.is_empty() || target.output.contains(&TargetType::M3u) || target.output.contains(&TargetType::Strm); - if needs_filename && target.filename.is_none() { - return create_m3u_filter_error_result!(M3uFilterErrorKind::Info, "filename is required for m3u type: {}", target.name); - } - if let Some(mapping_ids) = &target.mapping { let mut target_mappings = Vec::new(); for mapping_id in mapping_ids { diff --git a/src/model/model_config.rs b/src/model/model_config.rs index ec3760fb9..31b7b3721 100644 --- a/src/model/model_config.rs +++ b/src/model/model_config.rs @@ -31,7 +31,7 @@ pub(crate) fn default_as_empty_map() -> HashMap { HashMap::new() pub(crate) fn default_as_zero() -> u8 { 0 } -#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, Sequence, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Sequence, PartialEq, Eq, Hash)] pub(crate) enum TargetType { #[serde(rename = "m3u")] M3u, diff --git a/src/processing/playlist_processor.rs b/src/processing/playlist_processor.rs index fd482ee0d..6b56857c3 100644 --- a/src/processing/playlist_processor.rs +++ b/src/processing/playlist_processor.rs @@ -1,7 +1,7 @@ extern crate unidecode; use std::cell::RefCell; -use std::collections::{HashMap}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::thread; @@ -20,7 +20,7 @@ use crate::model::model_m3u::{FetchedPlaylist, FieldAccessor, PlaylistGroup, Pla use crate::model::stats::{InputStats, PlaylistStats}; use crate::processing::playlist_watch::process_group_watch; use crate::repository::m3u_repository::{write_m3u_playlist, write_strm_playlist}; -use crate::repository::xtream_repository::{write_xtream_playlist}; +use crate::repository::xtream_repository::write_xtream_playlist; fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { debug!("Filtering {} groups", playlist.len()); @@ -447,12 +447,12 @@ pub(crate) fn process_playlist(playlists: &mut [FetchedPlaylist], } fn persist_playlist(playlist: &[PlaylistGroup], - target: &ConfigTarget, cfg: &Config,) -> Result<(), Vec> { + target: &ConfigTarget, cfg: &Config, ) -> Result<(), Vec> { let mut errors = vec![]; - for target_type in &target.output { - match match target_type { - TargetType::M3u => write_m3u_playlist(target, cfg, playlist), - TargetType::Strm => write_strm_playlist(target, cfg, playlist), + for output in &target.output { + match match output.target { + TargetType::M3u => write_m3u_playlist(target, cfg, playlist, &output.filename), + TargetType::Strm => write_strm_playlist(target, cfg, playlist, &output.filename), TargetType::Xtream => write_xtream_playlist(target, cfg, playlist) } { Ok(_) => {} @@ -462,26 +462,7 @@ fn persist_playlist(playlist: &[PlaylistGroup], } } - // - // if target.filename.is_some() { - // let result = write_playlist(target, cfg, &mut new_playlist); - // match &result { - // Ok(..) => {} - // Err(e) => { - // error!("failed to write playlist {} {}", target.name, e); - // } - // } - // if !publish { - // return result; - // } - // } - // - // if publish { - // if let Some(TargetType::Xtream) = target.output { - // return xtream_save_playlist(target, cfg, &mut new_playlist); - // } - // } - Err(vec![M3uFilterError::new(M3uFilterErrorKind::Notify, format!("Persisting playlist failed: {}", &target.name))]) + if errors.is_empty() { Err(errors) } else { Ok(()) } } pub(crate) fn start_processing(cfg: Arc, targets: Arc) { @@ -495,6 +476,6 @@ pub(crate) fn start_processing(cfg: Arc, targets: Arc) { errors.iter().for_each(|err| error!("{}", err.message)); // send errors if let Some(message) = get_errors_notify_message!(errors, 255) { - send_message(&MsgKind::Error,&cfg.messaging, message.as_str()); + send_message(&MsgKind::Error, &cfg.messaging, message.as_str()); } } \ No newline at end of file diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 42d3c899a..47fd9eac3 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -81,21 +81,21 @@ fn kodi_style_rename(name: &String, style: &KodiStyle) -> String { String::from(name) } -pub(crate) fn get_m3u_file_path(cfg: &Config, target: &ConfigTarget) -> Option { - utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(&target.filename.as_ref().unwrap()))) +pub(crate) fn get_m3u_file_path(cfg: &Config, filename: &Option) -> Option { + utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(&filename.as_ref().unwrap()))) } -pub(crate) fn write_m3u_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { +pub(crate) fn write_m3u_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup], filename: &Option) -> Result<(), M3uFilterError> { macro_rules! cant_write_result { ($path:expr, $err:expr) => { create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err) } } if !new_playlist.is_empty() { - if target.filename.is_none() { + if filename.is_none() { return Err(M3uFilterError::new(M3uFilterErrorKind::Notify, "write m3u playlist failed: ".to_string())); } - if let Some(path) = get_m3u_file_path(cfg, target) { + if let Some(path) = get_m3u_file_path(cfg, filename) { match File::create(&path) { Ok(mut m3u_file) => { match check_write(m3u_file.write(b"#EXTM3U\n")) { @@ -123,16 +123,16 @@ pub(crate) fn write_m3u_playlist(target: &ConfigTarget, cfg: &Config, new_playli Ok(()) } -pub(crate) fn write_strm_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { +pub(crate) fn write_strm_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup], filename: &Option) -> Result<(), M3uFilterError> { if !new_playlist.is_empty() { - if target.filename.is_none() { + if filename.is_none() { return Err(M3uFilterError::new(M3uFilterErrorKind::Notify, "write strm playlist failed: ".to_string())); } let underscore_whitespace = target.options.as_ref().map_or(false, |o| o.underscore_whitespace); let cleanup = target.options.as_ref().map_or(false, |o| o.cleanup); let kodi_style = target.options.as_ref().map_or(false, |o| o.kodi_style); - if let Some(path) = utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(&target.filename.as_ref().unwrap()))) { + if let Some(path) = utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(&filename.as_ref().unwrap()))) { if cleanup { let _ = std::fs::remove_dir_all(&path); } diff --git a/src/utils.rs b/src/utils.rs index fc4083f7c..12f3d449c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,6 +1,4 @@ -use std::collections::HashSet; use std::fs; -use std::hash::Hash; use std::io::{BufRead, Write}; use std::path::PathBuf; use log::{debug, error}; @@ -250,7 +248,7 @@ pub(crate) fn get_input_json_content(input: &ConfigInput, url_str: &String, pers } } -pub(crate) fn dedup(v: &mut Vec) { - let mut uniques = HashSet::new(); - v.retain(|e| uniques.insert(*e)); -} \ No newline at end of file +// pub(crate) fn dedup(v: &mut Vec) { +// let mut uniques = HashSet::new(); +// v.retain(|e| uniques.insert(*e)); +// } \ No newline at end of file