diff --git a/README.md b/README.md index ea5724999..44c60b39e 100644 --- a/README.md +++ b/README.md @@ -1026,12 +1026,13 @@ It is whitespace-tolerant and uses familiar programming concepts with a custom s - Null value `null` - Regex Matching: `@FieldName ~ "Regex"` like in filter statements. You can match a `FieldName` or a existing `variable`. - Access a field in a regex match result: with `result.capture`. For example, if you have multiple captures you can access them by their name, or their index beginning at 1. -- Builtin String functions: +- Builtin functions: - concat(a, b, ...) - uppercase(a) - lowercase(a) - capitalize(a) - trim(a) + - number(a) - print(a) Example `print(uppercase("hello"))`. output is only visible in `trace` log level you can enable it like `log_level: debug,tuliprox::foundation::mapper=trace` in config - Assignment assigns an expression result. variable or field. @@ -1051,6 +1052,8 @@ result = match { } ``` - Map block assigns expression results to a variable or field + +Mapping over text It is possible to define multiple keys with `|` seperated for one case. ```dsl result = map variable_name { @@ -1065,6 +1068,17 @@ result = map variable_name { } ``` +Mapping over number ranges +```dls + year_group = map year { + ..2019 => "< 2020", + 2020..2023 => "2020 - 2023", + 2024..2025 => "2024 - 2025", + 2025.. => "> 2025", + _ => year_text, + } +``` + Example `mapping.yml` ```yaml @@ -1769,6 +1783,11 @@ The grace period means: if a user reaches the connection limit, we still allow o After a delay, we check whether old connections have been properly closed. If not, we then enforce the limit and terminate the excess connection(s). # Mapper example +## Grouping +We asume we have some groups with the text EU, SATELLITE, NATIONAL, NEWS, MUSIC, SPORT, RELIGION, FILM, KIDS, DOCU +in the group name. +We wwant to group the channels inside NEWS. NATIONAL, SATELLITE by their quality. +The other groups should get a number prefix for ordering. ```yaml group = Group ~ "(EU|SATELLITE|NATIONAL|NEWS|MUSIC|SPORT|RELIGION|FILM|KIDS|DOCU)" @@ -1910,4 +1929,32 @@ Update the playlist item ```dsl @Group = name @Caption = title_name -``` \ No newline at end of file +``` + +## Grouping by release year +We want to automatically group these channels by their release year, using the following logic: +- All movies released before 2020 should be grouped together under one label. +- Movies from 2020 onward should each be grouped by their specific year. +Example title: "Master Movie (2020)" +The result should look like +- FR | Movies < 2020 +- FR | Movies 2020 +- FR | Movies 2021 +- FR | Movies +```dsl +- filter: 'Group ~ "^FR" AND Caption ~ "\(?\d{4}\)?$"' + script: | + year_text = @Caption ~ "(\d{4})\)?$" + year = number(year_text) + year_group = map year { + ..2019 => "< 2020", + _ => year_text, + } + @Group = concat("FR | MOVIES ", year_group) +``` +Filter: Matches channels where the Group starts with "FR" and the Caption ends in a 4-digit year (optionally inside parentheses). +Regex extraction: Pulls the 4-digit year from the caption. +Mapping: + If the year is ≤ 2019, it maps to " < 2020". + Otherwise, the group is named by the actual year (e.g., "2021"). +Assignment: Constructs a new group label like "FR | MOVIES 2021" and assigns it to @Group \ No newline at end of file diff --git a/src/foundation/mapper.pest b/src/foundation/mapper.pest index 969019ecb..e5abad2df 100644 --- a/src/foundation/mapper.pest +++ b/src/foundation/mapper.pest @@ -4,12 +4,18 @@ null = { "null" } identifier = @{ !null ~ (ASCII_ALPHANUMERIC | "_")+ } var_access = { identifier ~ ("." ~ identifier)? } string_literal = @{ "\"" ~ ( "\\\"" | (!"\"" ~ ANY) )* ~ "\"" } +number = @{ "-"? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? } +number_range_from = { number ~ ".." } +number_range_to = { ".." ~ number } +number_range_full = { number ~ ".." ~ number } +number_range_eq = { number } +number_range = _{ number_range_full | number_range_from | number_range_to | number_range_eq} field = { ^"name" | ^"title" | ^"caption" | ^"group" | ^"id" | ^"chno" | ^"logo" | ^"logo_small" | ^"parent_code" | ^"audio_track" | ^"time_shift" | ^"rec" | ^"url" | ^"epg_channel_id" | ^"epg_id" } field_access = _{ "@" ~ field } regex_source = _{ field_access | identifier } regex_expr = { regex_source ~ regex_op ~ string_literal } -expression = { map_block | match_block | function_call | regex_expr | string_literal | var_access | field_access | null } -function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "trim" | "print" } +expression = { map_block | match_block | function_call | regex_expr | string_literal | number | var_access | field_access | null } +function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "trim" | "print" | "number" } function_call = { function_name ~ "(" ~ (expression ~ ("," ~ expression)*)? ~ ")" } any_match = { "_" } match_case_key = { any_match | identifier } @@ -17,7 +23,7 @@ match_case_key_list = { match_case_key ~ ("," ~ match_case_key)* } match_case = { match_case_key_list ~ "=>" ~ expression | "(" ~ match_case_key_list ~ ")" ~ "=>" ~ expression } match_block = { "match" ~ "{" ~ NEWLINE* ~ (match_case ~ ("," ~ NEWLINE* ~ match_case)*)? ~ ","? ~ NEWLINE* ~ "}" } map_case_key_list = { string_literal ~ ("|" ~ string_literal)* } -map_case_key = { any_match | map_case_key_list } +map_case_key = { any_match | number_range | map_case_key_list } map_case = { map_case_key ~ "=>" ~ expression } map_key = { identifier } map_block = { "map" ~ map_key ~ "{" ~ NEWLINE* ~ (map_case ~ ("," ~ NEWLINE* ~ map_case)*)? ~ ","? ~ NEWLINE* ~ "}" } diff --git a/src/foundation/mapper.rs b/src/foundation/mapper.rs index 352070e48..9a503e268 100644 --- a/src/foundation/mapper.rs +++ b/src/foundation/mapper.rs @@ -1,14 +1,15 @@ #![allow(clippy::empty_docs)] -use std::borrow::Cow; use crate::foundation::filter::ValueAccessor; -use crate::foundation::mapper::EvalResult::{AnyValue, Failure, Named, Undefined, Value}; +use crate::foundation::mapper::EvalResult::{AnyValue, Failure, Named, Number, Undefined, Value}; use crate::tuliprox_error::{create_tuliprox_error_result, info_err, TuliproxError, TuliproxErrorKind}; use crate::utils::Capitalize; use log::{debug, error, trace}; use pest::iterators::Pair; use pest::Parser; use regex::Regex; +use std::borrow::Cow; +use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; use std::str::FromStr; @@ -20,12 +21,18 @@ null = { "null" } identifier = @{ !null ~ (ASCII_ALPHANUMERIC | "_")+ } var_access = { identifier ~ ("." ~ identifier)? } string_literal = @{ "\"" ~ ( "\\\"" | (!"\"" ~ ANY) )* ~ "\"" } +number = @{ "-"? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? } +number_range_from = { number ~ ".." } +number_range_to = { ".." ~ number } +number_range_full = { number ~ ".." ~ number } +number_range_eq = { number } +number_range = _{ number_range_full | number_range_from | number_range_to | number_range_eq} field = { ^"name" | ^"title" | ^"caption" | ^"group" | ^"id" | ^"chno" | ^"logo" | ^"logo_small" | ^"parent_code" | ^"audio_track" | ^"time_shift" | ^"rec" | ^"url" | ^"epg_channel_id" | ^"epg_id" } field_access = _{ "@" ~ field } regex_source = _{ field_access | identifier } regex_expr = { regex_source ~ regex_op ~ string_literal } -expression = { map_block | match_block | function_call | regex_expr | string_literal | var_access | field_access | null } -function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "trim" | "print" } +expression = { map_block | match_block | function_call | regex_expr | string_literal | number | var_access | field_access | null } +function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "trim" | "print" | "number" } function_call = { function_name ~ "(" ~ (expression ~ ("," ~ expression)*)? ~ ")" } any_match = { "_" } match_case_key = { any_match | identifier } @@ -33,7 +40,7 @@ match_case_key_list = { match_case_key ~ ("," ~ match_case_key)* } match_case = { match_case_key_list ~ "=>" ~ expression | "(" ~ match_case_key_list ~ ")" ~ "=>" ~ expression } match_block = { "match" ~ "{" ~ NEWLINE* ~ (match_case ~ ("," ~ NEWLINE* ~ match_case)*)? ~ ","? ~ NEWLINE* ~ "}" } map_case_key_list = { string_literal ~ ("|" ~ string_literal)* } -map_case_key = { any_match | map_case_key_list } +map_case_key = { any_match | number_range | map_case_key_list } map_case = { map_case_key ~ "=>" ~ expression } map_key = { identifier } map_block = { "map" ~ map_key ~ "{" ~ NEWLINE* ~ (map_case ~ ("," ~ NEWLINE* ~ map_case)*)? ~ ","? ~ NEWLINE* ~ "}" } @@ -61,6 +68,10 @@ struct MatchCase { #[derive(Debug, Clone)] enum MapCaseKey { Text(String), + RangeFrom(f64), + RangeTo(f64), + RangeFull(f64, f64), + RangeEq(f64), AnyMatch, } @@ -84,6 +95,7 @@ enum BuiltInFunction { Capitalize, Trim, Print, + ToNumber, } impl FromStr for BuiltInFunction { @@ -97,6 +109,7 @@ impl FromStr for BuiltInFunction { "uppercase" => Ok(Self::Uppercase), "trim" => Ok(Self::Trim), "print" => Ok(Self::Print), + "number" => Ok(Self::ToNumber), _ => create_tuliprox_error_result!(TuliproxErrorKind::Info, "Unknown function {}", s), } } @@ -112,13 +125,14 @@ enum RegexSource { enum Expression { Identifier(String), StringLiteral(String), + NumberLiteral(f64), FieldAccess(String), VarAccess(String, String), RegexExpr { field: RegexSource, pattern: String, re_pattern: Regex }, FunctionCall { name: BuiltInFunction, args: Vec }, MatchBlock(Vec), MapBlock { key: MapKey, cases: Vec }, - NullValue + NullValue, } #[derive(Debug, Clone)] @@ -168,6 +182,9 @@ impl Statement { Value(content) => { setter.set(name, content.as_str()); } + Number(num) => { + setter.set(name, format_number(num).as_str()); + } Named(pairs) => { let mut result = String::with_capacity(128); for (i, (key, value)) in pairs.iter().enumerate() { @@ -213,7 +230,8 @@ impl MapperScript { } Expression::NullValue | Expression::FieldAccess(_) - | Expression::StringLiteral(_) => {} + | Expression::StringLiteral(_) + | Expression::NumberLiteral(_) => {} Expression::RegexExpr { field, pattern: _pattern, re_pattern: _re_pattern } => { match field { RegexSource::Identifier(ident) => { @@ -278,6 +296,14 @@ impl MapperScript { } case_keys.insert(value.as_str()); } + MapCaseKey::RangeEq(_) + | MapCaseKey::RangeTo(_) + | MapCaseKey::RangeFrom(_) => {} + MapCaseKey::RangeFull(from, to) => { + if *from > *to { + return create_tuliprox_error_result!(TuliproxErrorKind::Info, "Invalid range {from}..{to}"); + } + }, MapCaseKey::AnyMatch => { any_match_count += 1; if any_match_count > 1 { @@ -421,6 +447,27 @@ impl MapperScript { } Ok(matches) } + Rule::number_range_full => { + let mut inner = inner.into_inner(); + let start = inner.next().unwrap().as_str().parse::().unwrap(); + let end = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeFull(start, end)]) + } + Rule::number_range_from => { + let mut inner = inner.into_inner(); + let start = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeFrom(start)]) + } + Rule::number_range_to => { + let mut inner = inner.into_inner(); + let to = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeTo(to)]) + } + Rule::number_range_eq => { + let mut inner = inner.into_inner(); + let num = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeEq(num)]) + } Rule::any_match => Ok(vec![MapCaseKey::AnyMatch]), _ => create_tuliprox_error_result!(TuliproxErrorKind::Info, "Unexpected map key: {:?}", inner.as_rule()), } @@ -468,6 +515,15 @@ impl MapperScript { Ok(Expression::StringLiteral(content.to_string())) } + Rule::number => { + let raw = pair.as_str(); + if let Number(val) = to_number(raw) { + Ok(Expression::NumberLiteral(val)) + } else { + create_tuliprox_error_result!(TuliproxErrorKind::Info, "Invalid number {raw}") + } + } + Rule::regex_expr => { let mut inner = pair.into_inner(); let first = inner.next().unwrap(); @@ -563,17 +619,46 @@ impl Default for MapperContext { } } - -#[derive(Debug)] -#[derive(Clone)] +#[derive(Debug, Clone)] enum EvalResult { Undefined, Value(String), + Number(f64), Named(Vec<(String, String)>), AnyValue, Failure(String), } +fn to_number(value: &str) -> EvalResult { + match value.parse::() { + Ok(num) => Number(num), + Err(_) => Failure(format!("Invalid number: {value}")), + } +} + +fn compare_number(a: f64, b: f64) -> Ordering { + let epsilon = 1e-3; // = 0.001 + + if (a - b).abs() < epsilon { + Ordering::Equal + } else if a < b { + Ordering::Less + } else { + Ordering::Greater + } +} + +#[allow(clippy::cast_possible_truncation)] +fn format_number(num: f64) -> String { + let epsilon = 1e-3; // = 0.001 + + if num.fract().abs() < epsilon { + format!("{}", num as i64) + } else { + format!("{num}") + } +} + fn compare_tuple_vec<'a>( a: &'a [(String, String)], b: &'a [(String, String)], @@ -587,34 +672,74 @@ fn compare_tuple_vec<'a>( to_map(a) == to_map(b) } +fn match_number(num: f64, s: &str) -> bool { + if let Ok(val) = s.parse::() { + return compare_number(num, val) == Ordering::Equal; + } + false +} + +fn cmp_number(num: f64, s: &str) -> Option { + if let Ok(val) = s.parse::() { + return Some(compare_number(num, val)); + } + None +} + + impl EvalResult { fn matches(&self, other: &EvalResult) -> bool { match (self, other) { (AnyValue, _) | (_, AnyValue) => true, (Value(a), Value(b)) => a == b, + (Number(a), Value(b)) => match_number(*a, b), + (Value(a), Number(b)) => match_number(*b, a), + (Number(a), Number(b)) => compare_number(*a, *b) == Ordering::Equal, (Named(a), Named(b)) => compare_tuple_vec(a, b), _ => false, } } + fn compare(&self, other: &EvalResult) -> Option { + match (self, other) { + (AnyValue, _) | (_, AnyValue) => Some(Ordering::Equal), + (Value(a), Value(b)) => Some(a.cmp(b)), + (Number(a), Value(b)) => cmp_number(*a, b), + (Value(a), Number(b)) => match cmp_number(*b, a) { + None => None, + Some(ord) => { + match ord { + Ordering::Less => Some(Ordering::Greater), + Ordering::Equal => Some(Ordering::Equal), + Ordering::Greater => Some(Ordering::Less), + } + } + }, + (Number(a), Number(b)) => Some(compare_number(*a, *b)), + (Named(a), Named(b)) => if compare_tuple_vec(a, b) { Some(Ordering::Equal) } else { None }, + _ => None, + } + } + pub fn is_error(&self) -> bool { matches!(self, Failure(_)) } } -fn concat_args(args: &Vec) -> Vec<&str> { +fn concat_args(args: &Vec) -> Vec> { let mut result = vec![]; for arg in args { match arg { - Value(value) => result.push(value.as_str()), + Value(value) => result.push(Cow::Borrowed(value.as_str())), + Number(value) => result.push(Cow::Owned(format_number(*value))), Named(pairs) => { for (i, (key, value)) in pairs.iter().enumerate() { - result.push(key.as_str()); - result.push(": "); - result.push(value.as_str()); + result.push(Cow::Borrowed(key.as_str())); + result.push(Cow::Borrowed(": ")); + result.push(Cow::Borrowed(value.as_str())); if i < pairs.len() - 1 { - result.push(", "); + result.push(Cow::Borrowed(", ")); } } } @@ -649,7 +774,7 @@ impl Expression { None => Failure(format!("Variable with name {name} not found.")), Some(value) => match value { Undefined => Undefined, - Value(_) => Failure(format!("Variable with name {name} has no fields.")), + Number(_) | Value(_) => Failure(format!("Variable with name {name} has no fields.")), Named(values) => { for (key, val) in values { if key == field { @@ -663,6 +788,7 @@ impl Expression { } } Expression::StringLiteral(s) => Value(s.clone()), + Expression::NumberLiteral(num) => Number(*num), Expression::RegexExpr { field, pattern: _pattern, re_pattern } => { let source = match field { RegexSource::Identifier(ident) => { @@ -716,13 +842,22 @@ impl Expression { match name { BuiltInFunction::Concat => Value(concat_args(&evaluated_args).join("")), BuiltInFunction::Uppercase => Value(concat_args(&evaluated_args).join(" ").to_uppercase()), - BuiltInFunction::Trim => Value(concat_args(&evaluated_args).iter().map(|&s| s.trim()).collect::>().join(" ").trim().to_string()), + BuiltInFunction::Trim => Value(concat_args(&evaluated_args).iter().map(|s| s.trim()).collect::>().join(" ").trim().to_string()), BuiltInFunction::Lowercase => Value(concat_args(&evaluated_args).join(" ").to_lowercase()), - BuiltInFunction::Capitalize => Value(concat_args(&evaluated_args).iter().map(|&s| s.capitalize()).collect::>().join(" ")), + BuiltInFunction::Capitalize => Value(concat_args(&evaluated_args).iter().map(Capitalize::capitalize).collect::>().join(" ")), BuiltInFunction::Print => { trace!("[MapperScript] {}", concat_args(&evaluated_args).join("")); Undefined } + BuiltInFunction::ToNumber => { + let evaluated_arg = &evaluated_args[0]; + match evaluated_arg { + Value(value) => { + to_number(value) + } + _ => evaluated_arg.clone() + } + } } } } @@ -746,6 +881,7 @@ impl Expression { for case_key in case_keys { match case_key { Value(_) + | Number(_) | Named(_) | AnyValue => match_count += 1, Undefined | Failure(_) => {} @@ -773,6 +909,50 @@ impl Expression { if match key { MapCaseKey::Text(value) => key_value.matches(&Value(value.to_string())), MapCaseKey::AnyMatch => true, + MapCaseKey::RangeFrom(num) => { + match key_value.compare(&Number(*num)) { + None => false, + Some(ord) => match ord { + Ordering::Less => false, + Ordering::Equal | Ordering::Greater => true, + } + } + } + MapCaseKey::RangeTo(num) => { + match key_value.compare(&Number(*num)) { + None => false, + Some(ord) => match ord { + Ordering::Equal | Ordering::Less => true, + Ordering::Greater => false, + } + } + } + MapCaseKey::RangeFull(from, to) => { + match key_value.compare(&Number(*from)) { + None => false, + Some(ord) => match ord { + Ordering::Less => false, + Ordering::Equal | Ordering::Greater => { + match key_value.compare(&Number(*to)) { + None => false, + Some(ord) => match ord { + Ordering::Equal | Ordering::Less => true, + Ordering::Greater => false, + } + } + }, + } + } + } + MapCaseKey::RangeEq(num) => { + match key_value.compare(&Number(*num)) { + None => false, + Some(ord) => match ord { + Ordering::Equal => true, + Ordering::Less | Ordering::Greater => false, + } + } + } } { matches = true; break; diff --git a/src/utils/string_utils.rs b/src/utils/string_utils.rs index 776da5a59..98f7658ec 100644 --- a/src/utils/string_utils.rs +++ b/src/utils/string_utils.rs @@ -20,10 +20,12 @@ pub trait Capitalize { } // Implement the Capitalize trait for &str -impl Capitalize for &str { +impl> Capitalize for T { fn capitalize(&self) -> String { - let mut chars = self.chars(); - let first = chars.next() + let s = self.as_ref(); + let mut chars = s.chars(); + let first = chars + .next() .map(|c| c.to_uppercase().collect::()) .unwrap_or_default(); let rest = chars.as_str().to_lowercase(); @@ -31,13 +33,6 @@ impl Capitalize for &str { } } -// Implement the trait for String as well -impl Capitalize for String { - fn capitalize(&self) -> String { - self.as_str().capitalize() // Reuse the &str implementation - } -} - pub fn get_trimmed_string(value: &Option) -> Option { if let Some(v) = value { let trimmed = v.trim();