From d70a2a655adcd356ef08dc59a9bf002ee3e1c800 Mon Sep 17 00:00:00 2001 From: euzu Date: Sat, 14 Jan 2023 12:46:19 +0100 Subject: [PATCH] added tag captures for mapping --- CHANGELOG.md | 32 +++++++++++++++++++ README.md | 13 +++++++- src/config.rs | 11 ++----- src/m3u_processing.rs | 74 ++++++++++++++++++++++++++++++++----------- src/mapping.rs | 37 ++++++++++++++++------ src/model.rs | 23 +++++++++++--- 6 files changed, 147 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f0ec8b00..ff86f5d24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,12 @@ # Changelog +# v0.9.6(2023-01-14) +* Renamed `mappings.templates` attribute `key` to `name` +* `mappings.tag` is now a struct + - captures: List of captured variable names like `quality`. + - concat: if you have more than one captures defined this is the join string between them + - suffix: suffix for thge tag + - prefix: prefix for the tag + # v0.9.5(2023-01-13) * Upgraded libraries, fixed serde_yaml v.0.8 empty string bug. * Added Processing Pipe to target for filter, map and rename. Values are: @@ -12,6 +20,30 @@ default is Fmr * Added mapping parameter `match_as_ascii`. Default is `false`. If `true` before regexp matching the matching text will be converted to ascii. [unidecode](https://chowdhurya.github.io/rust-unidecode/unidecode/index.html) +Added regexp templates to mapper: +```yaml +mappings: + - id: France + tag: "" + match_as_ascii: true + templates: + - key: delimiter + value: '[\s_-]*' + - key: quality + value: '(?i)(?PHD|LQ|4K|UHD)?' + mapper: + - tvg_name: TF1 $quality + # https://regex101.com/r/UV233E/1 + tvg_names: + - '^\s*(FR)?[: |]?TF1!delimiter!!quality!\s*$' + tvg_id: TF1.fr + tvg_chno: "1" + tvg_logo: https://emojipedia-us.s3.amazonaws.com/source/skype/289/shrimp_1f990.png + group_title: + - FR + - TNT +``` + * `mapping` attribute for target is now a list. You can assign multiple mapper to a target. ``` mapping: diff --git a/README.md b/README.md index 40cd984f3..efb58ed28 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,12 @@ example mapping.yml file. ```yaml mappings: - id: France - tag: "" + tag: + captures: + - quality + concat: '|' + prefix: ' [ ' + suffix: ' ]' match_as_ascii: true templates: - key: delimiter @@ -133,6 +138,12 @@ mappings: - PLUS1 ``` +* `mappings.tag` is a struct + - captures: List of captured variable names like `quality`. The names should be equal to the regexp capture names. + - concat: if you have more than one captures defined this is the join string between them + - suffix: suffix for thge tag + - prefix: prefix for the tag + ## The EXTM3U format is an extension of the M3U format. m3u has become almost a standard for the formation of playlists of media players and media devices. diff --git a/src/config.rs b/src/config.rs index d07432ab1..f4a4017bf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,6 +7,8 @@ use crate::model::{ItemField, ProcessingOrder, SortOrder, TargetType}; use crate::utils; use crate::utils::get_working_path; +fn default_as_false() -> bool { false } +fn default_as_frm() -> ProcessingOrder { ProcessingOrder::Frm } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct ConfigSort { @@ -33,10 +35,6 @@ impl ConfigRename { } } -fn default_as_false() -> bool { - false -} - #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct ConfigOptions { #[serde(default = "default_as_false")] @@ -49,11 +47,6 @@ pub struct ConfigOptions { pub kodi_style: bool, } - -fn default_as_frm() -> ProcessingOrder { - ProcessingOrder::Frm -} - #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct ConfigTarget { pub filename: String, diff --git a/src/m3u_processing.rs b/src/m3u_processing.rs index fe5f3cd6b..0bb7d0333 100644 --- a/src/m3u_processing.rs +++ b/src/m3u_processing.rs @@ -7,7 +7,7 @@ use crate::{config, Config, get_playlist, m3u, utils}; use crate::model::SortOrder::{Asc, Desc}; use crate::filter::ValueProvider; use crate::m3u::{PlaylistGroup, PlaylistItem}; -use crate::mapping::Mapping; +use crate::mapping::{Mapping, MappingTag}; use crate::model::{ItemField, ProcessingOrder, TargetType}; @@ -25,15 +25,18 @@ fn check_write(res: std::io::Result) -> Result<(), std::io::Error> { } } -fn filter_playlist(playlist: &Vec, target: &ConfigTarget) -> Option> { +fn filter_playlist(playlist: &Vec, target: &ConfigTarget, verbose: bool) -> Option> { + if verbose { println!("Filtering {} groups", playlist.len()) } let mut new_playlist = Vec::new(); for pg in playlist { + if verbose { println!("Filtering group {} with {} items", pg.title, pg.channels.len()) } let mut channels = Vec::new(); for pli in &pg.channels { if is_valid(&pli, &target) { channels.push(pli.clone()); } } + if verbose { println!("Filtered group {} has now {} items", pg.title, channels.len()) } if channels.len() > 0 { new_playlist.push(PlaylistGroup { title: pg.title.clone(), @@ -44,8 +47,10 @@ fn filter_playlist(playlist: &Vec, target: &ConfigTarget) -> Opti Some(new_playlist) } -pub(crate) fn write_m3u(playlist: &Vec, target: &config::ConfigTarget, cfg: &config::Config) -> Result<(), std::io::Error> { - let pipe : Vec, target: &ConfigTarget) -> Option>> = match &target.processing_order { +pub(crate) fn write_m3u(playlist: &Vec, + target: &config::ConfigTarget, cfg: &config::Config, + verbose: bool) -> Result<(), std::io::Error> { + let pipe : Vec, target: &ConfigTarget, verbose: bool) -> Option>> = match &target.processing_order { ProcessingOrder::Frm => vec![filter_playlist, rename_playlist, map_playlist], ProcessingOrder::Fmr => vec![filter_playlist, map_playlist, rename_playlist], ProcessingOrder::Rfm => vec![rename_playlist, filter_playlist, map_playlist], @@ -54,9 +59,11 @@ pub(crate) fn write_m3u(playlist: &Vec, target: &config::Con ProcessingOrder::Mrf => vec![map_playlist, rename_playlist, filter_playlist] }; + if verbose { println!("Processing order is {}", &target.processing_order)} + let mut new_playlist= playlist.clone(); for f in pipe { - let r = f(&new_playlist, &target); + let r = f(&new_playlist, &target, verbose); if r.is_some() { new_playlist = r.unwrap(); } @@ -242,7 +249,7 @@ fn is_valid(pli: &m3u::PlaylistItem, target: &ConfigTarget) -> bool { return target.filter(&provider); } -fn exec_rename(pli: &mut m3u::PlaylistItem, rename: &Option>) { +fn exec_rename(pli: &mut m3u::PlaylistItem, rename: &Option>, verbose: bool) { match rename { Some(renames) => { if renames.len() > 0 { @@ -250,6 +257,7 @@ fn exec_rename(pli: &mut m3u::PlaylistItem, rename: &Option, target: &ConfigTarget) -> Option> { +fn rename_playlist(playlist: &Vec, target: &ConfigTarget, verbose: bool) -> Option> { match &target.rename { Some(renames) => { if renames.len() > 0 { @@ -270,6 +278,7 @@ fn rename_playlist(playlist: &Vec, target: &ConfigTarget) -> Opti match r.field { ItemField::Group => { let cap = r.re.as_ref().unwrap().replace_all(&grp.title, &r.new_name); + if verbose { println!("Renamed group {} to {}", &grp.title, cap); } grp.title = cap.into_owned(); } _ => {} @@ -277,7 +286,7 @@ fn rename_playlist(playlist: &Vec, target: &ConfigTarget) -> Opti } for pli in &mut grp.channels { - exec_rename(pli, &target.rename) + exec_rename(pli, &target.rename, verbose) } new_playlist.push(grp); } @@ -289,15 +298,25 @@ fn rename_playlist(playlist: &Vec, target: &ConfigTarget) -> Opti } } -fn map_channel(channel: &PlaylistItem, mapping: &Mapping) -> PlaylistItem { - if mapping.mapper.len() > 0 { - let tag = if mapping.tag.is_empty() { - String::from("") - } else { - format!("- {}", &mapping.tag) - }; +fn get_mapping_tag(mapping: &&Mapping) -> MappingTag { + let mapping_tag = match &mapping.tag { + Some(mtag) => mtag.clone(), + _ => MappingTag { + captures: vec![], + concat: "".to_string(), + prefix: "".to_string(), + suffix: "".to_string(), + } + }; + mapping_tag +} +fn map_channel(channel: &PlaylistItem, mapping: &Mapping, verbose: bool) -> PlaylistItem { + if mapping.mapper.len() > 0 { let channel_name = if mapping.match_as_ascii { unidecode(&channel.header.name) } else { String::from(&channel.header.name) }; + if verbose && mapping.match_as_ascii { println!("Decoded {} for matching to {}", &channel.header.name, &channel_name)}; + let mut tag = "".to_string(); + let mapping_tag = get_mapping_tag(&mapping); for m in &mapping.mapper { match &m._re { @@ -318,10 +337,27 @@ fn map_channel(channel: &PlaylistItem, mapping: &Mapping) -> PlaylistItem { "" }; chan_name = String::from(chan_name.replace(format!("${}", cname.as_str()).as_str(), repl)); + + for c in &mapping_tag.captures { + if c.eq(cname) { + if tag.is_empty() { + tag = String::from(repl); + } else { + tag = format!("{}{}{}", tag, &mapping_tag.concat, String::from(repl)) + } + } + } } } } - chan.header.name = format!("{}{}", &chan_name, tag); + + if !tag.is_empty() { + tag = format!("{}{}{}", mapping_tag.prefix, tag, mapping_tag.suffix) + } + + let new_name = format!("{}{}", &chan_name, tag); + if verbose { println!("Mapped {} to {}", &channel.header.name, new_name)} + chan.header.name = new_name; chan.header.chno = m.tvg_chno.clone(); chan.header.id = m.tvg_id.clone(); chan.header.logo = m.tvg_logo.clone(); @@ -339,7 +375,7 @@ fn map_channel(channel: &PlaylistItem, mapping: &Mapping) -> PlaylistItem { return channel.clone(); } -fn map_playlist(playlist: &Vec, target: &ConfigTarget) -> Option> { +fn map_playlist(playlist: &Vec, target: &ConfigTarget, verbose: bool) -> Option> { if target._mapping.is_some() { let mut new_playlist: Vec = Vec::new(); for g in playlist { @@ -347,7 +383,7 @@ fn map_playlist(playlist: &Vec, target: &ConfigTarget) -> Option< let mappings = target._mapping.as_ref().unwrap(); for mapping in mappings { if mapping.mapper.len() > 0 { - grp.channels = grp.channels.iter().map(|chan| map_channel(&chan, &mapping)).collect(); + grp.channels = grp.channels.iter().map(|chan| map_channel(&chan, &mapping, verbose)).collect(); } } new_playlist.push(grp); @@ -390,7 +426,7 @@ pub fn process_targets(cfg: &Config, verbose: bool) { match &result { Some(playlist) => { for target in source.targets.iter() { - match write_m3u(playlist, target, &cfg) { + match write_m3u(playlist, target, &cfg, verbose) { Ok(_) => (), Err(e) => println!("Failed to write file: {}", e) } diff --git a/src/mapping.rs b/src/mapping.rs index c4779f16b..b26f87e7b 100644 --- a/src/mapping.rs +++ b/src/mapping.rs @@ -1,3 +1,6 @@ +fn default_as_false() -> bool { false } +fn default_as_empty_str() -> String { String::from("") } + #[derive(Debug)] pub struct MapperRe { pub re: regex::Regex, @@ -32,7 +35,7 @@ impl Mapper { let mut regstr = String::from(rs); let templ : &Vec = templates.unwrap(); for t in templ { - regstr = regstr.replace(format!("!{}!", &t.key).as_str(), &t.value); + regstr = regstr.replace(format!("!{}!", &t.name).as_str(), &t.value); } let re = regex::Regex::new(regstr.as_str()); @@ -66,31 +69,47 @@ impl Clone for Mapper { } } - -fn default_as_false() -> bool { - false -} - #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct MapperTemplate { - pub key: String, + pub name: String, pub value: String, } impl Clone for MapperTemplate { fn clone(&self) -> Self { MapperTemplate { - key: self.key.clone(), + name: self.name.clone(), value: self.value.clone() } } } +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct MappingTag { + pub captures: Vec, + #[serde(default = "default_as_empty_str")] + pub concat: String, + #[serde(default = "default_as_empty_str")] + pub prefix: String, + #[serde(default = "default_as_empty_str")] + pub suffix: String, +} + +impl Clone for MappingTag { + fn clone(&self) -> Self { + MappingTag { + captures: self.captures.clone(), + concat: self.prefix.clone(), + prefix: self.prefix.clone(), + suffix: self.suffix.clone(), + } + } +} #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct Mapping { pub id: String, - pub tag: String, + pub tag: Option, #[serde(default = "default_as_false")] pub match_as_ascii: bool, pub templates: Option>, diff --git a/src/model.rs b/src/model.rs index 9431d5412..089c577c3 100644 --- a/src/model.rs +++ b/src/model.rs @@ -6,11 +6,6 @@ pub enum TargetType { Strm, } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Sequence)] -pub enum ProcessingOrder { - Frm, Fmr, Rfm, Rmf, Mfr, Mrf -} - impl std::fmt::Display for TargetType { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match *self { @@ -20,6 +15,24 @@ impl std::fmt::Display for TargetType { } } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Sequence)] +pub enum ProcessingOrder { + Frm, Fmr, Rfm, Rmf, Mfr, Mrf +} + +impl std::fmt::Display for ProcessingOrder { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + ProcessingOrder::Frm => write!(f, "filter, rename, map"), + ProcessingOrder::Fmr => write!(f, "filter, map, rename"), + ProcessingOrder::Rfm => write!(f, "rename, filter, map"), + ProcessingOrder::Rmf => write!(f, "rename, map, filter"), + ProcessingOrder::Mfr => write!(f, "map, filter, rename"), + ProcessingOrder::Mrf => write!(f, "map, rename, filter"), + } + } +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Sequence)] pub enum ItemField { Group,