From 76e96093d3dc00fe9cef4984007c2450e78628d8 Mon Sep 17 00:00:00 2001 From: euzu Date: Mon, 23 Oct 2023 18:12:24 +0200 Subject: [PATCH] mappings regexp replacing in attributes --- CHANGELOG.md | 1 + README.md | 29 ++++++++++++ docker/Dockerfile | 2 + docker/docker-compose.yml | 3 ++ src/filter.rs | 4 +- src/model/mapping.rs | 40 ++++++++++------ src/model/model_config.rs | 2 +- src/model/model_m3u.rs | 70 ++++++++++++++++++++++------ src/processing/m3u_parser.rs | 7 +-- src/processing/playlist_processor.rs | 11 ++--- src/processing/xtream_parser.rs | 10 ++-- src/repository/m3u_repository.rs | 7 +-- src/repository/xtream_repository.rs | 4 +- 13 files changed, 140 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4824c68bc..797be1560 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ * Fixed TLS problem with docker scratch * Added simple stats * Target Output is now a list of multiple output formats, !breaking change! +* RegExp captures can now be used in mapper attributes Changes in `config.yml` ```yaml diff --git a/README.md b/README.md index 247391c76..610b6feaf 100644 --- a/README.md +++ b/README.md @@ -470,6 +470,29 @@ Attributes is a map of key value pairs. Valid keys are: - `source` If the regexps matches, the given fields will be set to the new value +You can use `captures` in attributes. +For example you want to `rewrite` the `base_url` for channels in a specific group. + +```yaml + +mappings: + templates: + - name: sports + value: 'Group ~ ".*SPORT.*"' + - name: source + value: 'Url ~ "https?:\/\/(.*?)\/(?P.*)$"' + + mapping: + - id: sport-mapper + mapper: + - filter: '!sports!' + pattern: "!source!" + attributes: + url: http://my.bubble-gum.tv/ +``` + +In this example all channels the urls of all channels with a group name containing `SPORT` will be changed. + #### 2.3.4.4 `suffix` Suffix is a map of key value pairs. Valid keys are @@ -537,6 +560,8 @@ mappings: value: '[\s_-]*' - name: quality value: '(?i)(?PHD|LQ|4K|UHD)?' + - name: source + value: 'Url ~ "https?:\/\/(.*?)\/(?P.*)$"' tags: - name: quality captures: @@ -548,6 +573,10 @@ mappings: - id: France match_as_ascii: true mapper: + - filter: 'Name ~ "^TF.*"' + pattern: '!source!' + attributes: + url: http://my.iptv.proxy.com/ - pattern: 'Name ~ "^TF1$"' attributes: name: TF1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 5ebaa47cc..9d9d62322 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,6 +5,8 @@ FROM scratch WORKDIR / COPY ./m3u-filter / COPY ./config.yml / +COPY ./api-proxy.yml / +COPY ./mapping.yml / COPY ./web /web # the tls certificates: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 62560396e..efb9a37d6 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -6,6 +6,9 @@ services: working_dir: / volumes: - ./data:/data + # - ./config.yml:/config.yml + # - ./api-proxy.yml:/api-proxy.yml + # - ./mapping.yml:/mapping.yml ports: - "8901:8901" restart: unless-stopped diff --git a/src/filter.rs b/src/filter.rs index 74be8c969..64617c6b2 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -18,7 +18,7 @@ pub(crate) fn get_field_value(pli: &PlaylistItem, field: &ItemField) -> String { ItemField::Group => header.group.as_str(), ItemField::Name => header.name.as_str(), ItemField::Title => header.title.as_str(), - ItemField::Url => pli.url.as_str(), + ItemField::Url => header.url.as_str(), }; String::from(value) } @@ -29,7 +29,7 @@ pub(crate) fn set_field_value(pli: &mut PlaylistItem, field: &ItemField, value: ItemField::Group => header.group = value, ItemField::Name => header.name = value, ItemField::Title => header.title = value, - ItemField::Url => {} + ItemField::Url => header.url = value, }; } diff --git a/src/model/mapping.rs b/src/model/mapping.rs index d5644bb4a..8dfe15c04 100644 --- a/src/model/mapping.rs +++ b/src/model/mapping.rs @@ -1,13 +1,15 @@ use std::cell::RefCell; use std::collections::HashMap; + use log::{debug, error}; use regex::Regex; + +use crate::{handle_m3u_filter_error_result, valid_property}; use crate::filter::{Filter, get_filter, PatternTemplate, prepare_templates, RegexWithCaptures, ValueProcessor}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; +use crate::model::model_config::{AFFIX_FIELDS, default_as_empty_map, default_as_empty_str, + default_as_false, ItemField, MAPPER_ATTRIBUTE_FIELDS, }; use crate::model::model_m3u::{FieldAccessor, PlaylistItem}; -use crate::model::model_config::{ItemField, MAPPER_ATTRIBUTE_FIELDS, AFFIX_FIELDS, - default_as_empty_str, default_as_false, default_as_empty_map, }; -use crate::{handle_m3u_filter_error_result, valid_property}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct MappingTag { @@ -41,6 +43,8 @@ pub(crate) struct Mapper { pub _tags: Vec, #[serde(skip_serializing, skip_deserializing)] pub _tagre: Option, + #[serde(skip_serializing, skip_deserializing)] + pub _attre: Option, } impl Mapper { @@ -62,6 +66,7 @@ impl Mapper { _ => vec![] }; self._tagre = Some(Regex::new("").unwrap()); + self._attre = Some(Regex::new("<(.*?)>").unwrap()); Ok(()) } Err(err) => Err(err) @@ -79,25 +84,34 @@ impl MappingValueProcessor<'_> { self.pli.borrow().header.borrow().get_field(key).map(String::from) } - fn set_property(&mut self, key: &str, value: &String) { + fn set_property(&mut self, key: &str, value: &str) { if !self.pli.borrow().header.borrow_mut().set_field(key, value) { error!("Cant set unknown field {} to {}", key, value); } debug!("Property {} set to {}", key, value); } - fn apply_attributes(&mut self) { + fn apply_attributes(&mut self, captured_names: &HashMap<&str, &str>) { let mapper = self.mapper.borrow(); + let attr_re = &mapper._attre.as_ref().unwrap(); let attributes = &mapper.attributes; drop(mapper); for (key, value) in attributes { if valid_property!(key.as_str(), MAPPER_ATTRIBUTE_FIELDS) { - self.set_property(key, value); + if value.contains('<') { // possible replacement + let replaced = attr_re.replace_all(value, |captures: ®ex::Captures| { + let capture_name = &captures[1]; + captured_names.get(&capture_name).unwrap_or(&&captures[0]).to_string() + }); + self.set_property(key, &replaced); + } else { + self.set_property(key, value); + } } } } - fn apply_tags(&mut self, value: &String, captures: &HashMap<&String, &str>) -> Option { + fn apply_tags(&mut self, value: &String, captures: &HashMap<&str, &str>) -> Option { let mut new_value = String::from(value); let tag_captures = self.mapper.borrow()._tagre.as_ref().unwrap().captures_iter(value) .filter(|caps| caps.len() > 1) @@ -111,7 +125,7 @@ impl MappingValueProcessor<'_> { // we have the right tag, now get all captured values let mut captured_tag_values: Vec<&str> = Vec::new(); for cap in &mapping_tag.captures { - match captures.get(&cap) { + match captures.get(cap.as_str()) { Some(cap_value) => captured_tag_values.push(cap_value), _ => { debug!("Cant find any tag match for {}", tag_capture); @@ -136,7 +150,7 @@ impl MappingValueProcessor<'_> { Some(new_value) } - fn apply_suffix(&mut self, captures: &HashMap<&String, &str>) { + fn apply_suffix(&mut self, captures: &HashMap<&str, &str>) { let mapper = self.mapper.borrow(); let suffix = &mapper.suffix; drop(mapper); @@ -153,7 +167,7 @@ impl MappingValueProcessor<'_> { } } - fn apply_prefix(&mut self, captures: &HashMap<&String, &str>) { + fn apply_prefix(&mut self, captures: &HashMap<&str, &str>) { let mapper = self.mapper.borrow(); let prefix = &mapper.prefix; drop(mapper); @@ -199,11 +213,11 @@ impl ValueProcessor for MappingValueProcessor<'_> { "" }; debug!("match {}: {}", capture_name, capture_value); - captured_values.insert(capture_name, capture_value); + captured_values.insert(capture_name.as_str(), capture_value); } ); } - let _ = &MappingValueProcessor::<'_>::apply_attributes(self); + let _ = &MappingValueProcessor::<'_>::apply_attributes(self, &captured_values); let _ = &MappingValueProcessor::<'_>::apply_suffix(self, &captured_values); let _ = &MappingValueProcessor::<'_>::apply_prefix(self, &captured_values); let _ = &MappingValueProcessor::<'_>::apply_assignments(self); @@ -247,7 +261,6 @@ impl MappingDefinition { } Err(err) => return Err(err), } - }; for mapping in &mut self.mapping { let template_list = match &self.templates { @@ -283,4 +296,3 @@ impl Mappings { None } } - diff --git a/src/model/model_config.rs b/src/model/model_config.rs index 31b7b3721..17a3c6188 100644 --- a/src/model/model_config.rs +++ b/src/model/model_config.rs @@ -8,7 +8,7 @@ pub(crate) const MAPPER_ATTRIBUTE_FIELDS: &[&str] = &[ "audio_track", "time_shift", "rec", - "source", + "url", ]; pub(crate) const AFFIX_FIELDS: &[&str] = &["name", "title", "group"]; diff --git a/src/model/model_m3u.rs b/src/model/model_m3u.rs index 2b00352b3..3d83cea75 100644 --- a/src/model/model_m3u.rs +++ b/src/model/model_m3u.rs @@ -1,6 +1,8 @@ use std::cell::RefCell; + use serde::{Deserialize, Serialize}; use serde_json::Value; + use crate::model::config::{ConfigInput, ConfigOptions}; // https://de.wikipedia.org/wiki/M3U @@ -16,7 +18,7 @@ pub(crate) struct FetchedPlaylist<'a> { pub(crate) enum XtreamCluster { Live = 1, Video = 2, - Series = 3 + Series = 3, } pub(crate) fn default_stream_cluster() -> XtreamCluster { XtreamCluster::Live } @@ -39,6 +41,8 @@ pub(crate) struct PlaylistItemHeader { pub time_shift: String, pub rec: String, pub source: String, + // this is the source content not the url + pub url: String, #[serde(default = "default_stream_cluster", skip_serializing, skip_deserializing)] pub xtream_cluster: XtreamCluster, #[serde(skip_serializing, skip_deserializing)] @@ -59,6 +63,7 @@ impl FieldAccessor for PlaylistItemHeader { "time_shift" => Some(&self.time_shift), "rec" => Some(&self.rec), "source" => Some(&self.source), + "url" => Some(&self.url), _ => None } } @@ -66,17 +71,54 @@ impl FieldAccessor for PlaylistItemHeader { fn set_field(&mut self, field: &str, value: &str) -> bool { let val = String::from(value); match field { - "id" => { self.id = val; true}, - "name" => { self.name = val; true } - "logo" => { self.logo = val; true } - "logo_small" => { self.logo_small = val; true } - "group" => { self.group = val; true } - "title" => { self.title = val; true } - "parent_code" => { self.parent_code = val; true } - "audio_track" => { self.audio_track = val; true } - "time_shift" => { self.time_shift = val; true } - "rec" => { self.rec = val; true } - "source" => { self.source = val; true } + "id" => { + self.id = val; + true + } + "name" => { + self.name = val; + true + } + "logo" => { + self.logo = val; + true + } + "logo_small" => { + self.logo_small = val; + true + } + "group" => { + self.group = val; + true + } + "title" => { + self.title = val; + true + } + "parent_code" => { + self.parent_code = val; + true + } + "audio_track" => { + self.audio_track = val; + true + } + "time_shift" => { + self.time_shift = val; + true + } + "rec" => { + self.rec = val; + true + } + "source" => { + self.source = val; + true + } + "url" => { + self.url = val; + true + } _ => false } } @@ -85,7 +127,6 @@ impl FieldAccessor for PlaylistItemHeader { #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct PlaylistItem { pub header: RefCell, - pub url: String, } impl PlaylistItem { @@ -116,10 +157,11 @@ impl PlaylistItem { if !header.rec.is_empty() { line = format!("{} rec=\"{}\"", line, header.rec); } - format!("{},{}\n{}", line, header.title, self.url) + format!("{},{}\n{}", line, header.title, header.url) } } + #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct PlaylistGroup { pub id: i32, diff --git a/src/processing/m3u_parser.rs b/src/processing/m3u_parser.rs index 9818b37ae..56da3249d 100644 --- a/src/processing/m3u_parser.rs +++ b/src/processing/m3u_parser.rs @@ -49,7 +49,7 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { } } -fn create_empty_playlistitem_header(content: &String) -> PlaylistItemHeader { +fn create_empty_playlistitem_header(content: &String, url: String) -> PlaylistItemHeader { PlaylistItemHeader { id: String::from(""), name: String::from(""), @@ -62,13 +62,14 @@ fn create_empty_playlistitem_header(content: &String) -> PlaylistItemHeader { time_shift: String::from(""), rec: String::from(""), source: String::from(content), + url, xtream_cluster: XtreamCluster::Live, additional_properties: None, } } fn process_header(video_suffixes: &Vec<&str>,content: &String, url: String) -> PlaylistItemHeader { - let mut plih = create_empty_playlistitem_header(content); + let mut plih = create_empty_playlistitem_header(content, url.clone()); let mut it = content.chars(); let line_token = token_till(&mut it, ':'); if line_token == Some(String::from("#EXTINF")) { @@ -139,7 +140,7 @@ pub(crate) fn parse_m3u(cfg: &Config, lines: &Vec) -> Vec continue; } if let Some(header_value) = header { - let item = PlaylistItem { header: RefCell::new(process_header(&video_suffixes, &header_value, String::from(line))), url: String::from(line) }; + let item = PlaylistItem { header: RefCell::new(process_header(&video_suffixes, &header_value, String::from(line))) }; if let Some(group_value) = group { if item.header.borrow().group.is_empty() { item.header.borrow_mut().group = group_value; diff --git a/src/processing/playlist_processor.rs b/src/processing/playlist_processor.rs index 390ebef08..1180dd1be 100644 --- a/src/processing/playlist_processor.rs +++ b/src/processing/playlist_processor.rs @@ -192,18 +192,17 @@ macro_rules! apply_pattern { }}; } -fn map_channel(channel: &PlaylistItem, mapping: &Mapping) -> PlaylistItem { +fn map_channel(channel: PlaylistItem, mapping: &Mapping) -> PlaylistItem { if !mapping.mapper.is_empty() { let header = channel.header.borrow(); let channel_name = if mapping.match_as_ascii { unidecode(&header.name) } else { String::from(&header.name) }; if mapping.match_as_ascii { debug!("Decoded {} for matching to {}", &header.name, &channel_name); }; drop(header); - let ref_chan = RefCell::new(channel); + let ref_chan = RefCell::new(&channel); let provider = ValueProvider { pli: ref_chan.clone() }; let mut mock_processor = MockValueProcessor {}; - for m in &mapping.mapper { - let mut processor = MappingValueProcessor { pli: ref_chan.clone(), mapper: RefCell::new(m) }; + let mut processor = MappingValueProcessor { pli: ref_chan.clone(), mapper: RefCell::new(m)}; match &m._filter { Some(filter) => { if filter.filter(&provider, &mut mock_processor) { @@ -216,7 +215,7 @@ fn map_channel(channel: &PlaylistItem, mapping: &Mapping) -> PlaylistItem { }; } } - channel.clone() + channel } fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option> { @@ -225,7 +224,7 @@ fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option let mut grp = playlist_group.clone(); let mappings = target._mapping.as_ref().unwrap(); mappings.iter().filter(|mapping| !mapping.mapper.is_empty()).for_each(|mapping| - grp.channels = grp.channels.iter_mut().map(|chan| map_channel(chan, mapping)).collect()); + grp.channels = grp.channels.drain(..).map(|chan| map_channel(chan, mapping)).collect()); grp }).collect(); diff --git a/src/processing/xtream_parser.rs b/src/processing/xtream_parser.rs index f90ac1b69..ec98510e4 100644 --- a/src/processing/xtream_parser.rs +++ b/src/processing/xtream_parser.rs @@ -222,7 +222,7 @@ impl XtreamStream { } } -fn process_category(category: &serde_json::Value) -> Result, M3uFilterError> { +fn process_category(category: &Value) -> Result, M3uFilterError> { match serde_json::from_value::>(category.to_owned()) { Ok(category_list) => Ok(category_list), Err(err) => { @@ -232,7 +232,7 @@ fn process_category(category: &serde_json::Value) -> Result, } -fn process_streams(xtream_cluster: &XtreamCluster, streams: &serde_json::Value) -> Result, M3uFilterError> { +fn process_streams(xtream_cluster: &XtreamCluster, streams: &Value) -> Result, M3uFilterError> { match serde_json::from_value::>(streams.to_owned()) { Ok(stream_list) => Ok(stream_list), Err(err) => { @@ -242,7 +242,7 @@ fn process_streams(xtream_cluster: &XtreamCluster, streams: &serde_json::Value) } pub(crate) fn parse_xtream(cat_id_cnt: &AtomicI32, xtream_cluster: &XtreamCluster, - category: &serde_json::Value, streams: &serde_json::Value, + category: &Value, streams: &Value, stream_base_url: &String) -> Result>, M3uFilterError> { match process_category(category) { Ok(mut categories) => { @@ -269,11 +269,11 @@ pub(crate) fn parse_xtream(cat_id_cnt: &AtomicI32, xtream_cluster: &XtreamCluste audio_track: "".to_string(), time_shift: "".to_string(), rec: "".to_string(), - source: String::from(&stream.direct_source), + source: "".to_string(), // String::from(&stream.direct_source), + url: if stream.direct_source.is_empty() { format!("{}/{}", stream_base_url, stream.get_stream_id())} else { String::from(&stream.direct_source) }, xtream_cluster: xtream_cluster.clone(), additional_properties: stream.get_additional_properties(), }), - url: format!("{}/{}", stream_base_url, stream.get_stream_id()), }; grp.add(item); } diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 47fd9eac3..d8ede6c27 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -142,12 +142,13 @@ pub(crate) fn write_strm_playlist(target: &ConfigTarget, cfg: &Config, new_playl }; for pg in new_playlist { for pli in &pg.channels { - let dir_path = path.join(sanitize_for_filename(&pli.header.borrow().group, underscore_whitespace)); + let header = &pli.header.borrow(); + let dir_path = path.join(sanitize_for_filename(&header.group, underscore_whitespace)); if let Err(e) = std::fs::create_dir_all(&dir_path) { error!("cant create directory: {:?}", &path); return create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write strm playlist: {}", e); }; - let mut file_name = sanitize_for_filename(&pli.header.borrow().title, underscore_whitespace); + let mut file_name = sanitize_for_filename(&header.title, underscore_whitespace); if kodi_style { let style = KodiStyle { season: regex::Regex::new(r"[Ss]\d\d").unwrap(), @@ -160,7 +161,7 @@ pub(crate) fn write_strm_playlist(target: &ConfigTarget, cfg: &Config, new_playl let file_path = dir_path.join(format!("{}.strm", file_name)); match File::create(&file_path) { Ok(mut strm_file) => { - match check_write(strm_file.write(pli.url.as_bytes())) { + match check_write(strm_file.write(header.url.as_bytes())) { Ok(_) => (), Err(e) => return create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write strm playlist: {}", e), } diff --git a/src/repository/xtream_repository.rs b/src/repository/xtream_repository.rs index 9401fe348..61df2d5b4 100644 --- a/src/repository/xtream_repository.rs +++ b/src/repository/xtream_repository.rs @@ -126,13 +126,13 @@ pub(crate) fn write_xtream_playlist(target: &ConfigTarget, cfg: &Config, playlis match header.xtream_cluster { XtreamCluster::Live => { document.insert("stream_id".to_string(), stream_id); - document.insert("direct_source".to_string(), Value::String(header.source.clone())); + document.insert("direct_source".to_string(), Value::String(header.url.clone())); document.insert("thumbnail".to_string(), Value::String(header.logo_small.clone())); document.insert("custom_sid".to_string(), Value::String("".to_string())); } XtreamCluster::Video => { document.insert("stream_id".to_string(), stream_id); - document.insert("direct_source".to_string(), Value::String(header.source.clone())); + document.insert("direct_source".to_string(), Value::String(header.url.clone())); document.insert("custom_sid".to_string(), Value::String("".to_string())); } XtreamCluster::Series => {