euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-16 04:42:18 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

serving json files for xtream

Browse Source
This commit is contained in:
euzu
2023-10-08 19:54:59 +02:00
parent 62960ddeb8
commit 9e2c8cfb83
18 changed files with 370 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+25 -7
View File
@@ -592,6 +592,19 @@ dependencies = [
"syn 1.0.107",
]
[[package]]
name = "env_logger"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
dependencies = [
"humantime",
"is-terminal",
"log",
"regex",
"termcolor",
]
[[package]]
name = "errno"
version = "0.2.8"
@@ -864,6 +877,12 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.24"
@@ -1020,9 +1039,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.139"
version = "0.2.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
[[package]]
name = "link-cplusplus"
@@ -1069,12 +1088,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.17"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "m3u-filter"
@@ -1089,7 +1105,9 @@ dependencies = [
"clap",
"cron",
"enum-iterator",
"env_logger",
"futures",
"log",
"mime",
"openssl",
"path-absolutize",
Cargo.toml
+2
View File
@@ -35,3 +35,5 @@ unidecode = "0"
petgraph = "0"
openssl = { version = "*", features = ["vendored"] } #https://docs.rs/openssl/0.10.34/openssl/#vendored
mime = "0.3.17"
log = "0.4.20"
env_logger = "0.10.0"
README.md
+14 -2
View File
@@ -487,7 +487,19 @@ mappings:
group: '|FR|TNT'
```
## 3. Compilation
## 3. User
If you use the proxy functionality,
you need to create a `user.yml` configuration.
```yaml
user:
- target: pl1
credentials:
- {username: xtream, password: xtream.secret}
```
## 4. Compilation
### Static binary for docker
#### Install prerequisites
@@ -555,7 +567,7 @@ Compile it with:
cargo build --release --target x86_64-pc-windows-gnu
```
## 4. The EXTM3U format is an extension of the M3U format.
## 5. The EXTM3U format is an extension of the M3U format.
m3u has become almost a standard for the formation of playlists of media players and media devices.
A file in the EXTM3U format is a text file with the extension m3u or m3u8.
src/api.rs
+6 -7
View File
@@ -11,9 +11,10 @@ use actix_web::web::Data;
use crate::model_api::{AppState, PlaylistRequest, ServerConfig};
use crate::config::{Config, ConfigInput, InputType, ProcessTargets};
use crate::playlist_processor;
use crate::{exit, playlist_processor};
use crate::download::get_m3u_playlist;
use crate::xtream_player_api::xtream_player_api;
use log::{error};
#[get("/")]
async fn index(
@@ -41,7 +42,7 @@ pub(crate) async fn playlist(
suffix: None,
enabled: true,
};
let result = get_m3u_playlist(&_app_state.config, &input, &_app_state.config.working_dir, false);
let result = get_m3u_playlist(&_app_state.config, &input, &_app_state.config.working_dir);
HttpResponse::Ok().json(result)
}
@@ -56,14 +57,13 @@ pub(crate) async fn config(
}
#[actix_web::main]
pub(crate) async fn start_server(cfg: Config, targets: ProcessTargets, verbose: bool) -> futures::io::Result<()> {
pub(crate) async fn start_server(cfg: Config, targets: ProcessTargets) -> futures::io::Result<()> {
let host = cfg.api.host.clone();
let port = cfg.api.port;
let web_dir = cfg.api.web_root.clone();
let web_dir_path = PathBuf::from(&web_dir);
if !web_dir_path.exists() || !web_dir_path.is_dir() {
println!("web_root does not exists or is not an directory: {:?}", &web_dir_path);
std::process::exit(1);
exit!("web_root does not exists or is not an directory: {:?}", &web_dir_path);
}
let schedule = cfg.schedule.clone();
@@ -72,7 +72,6 @@ pub(crate) async fn start_server(cfg: Config, targets: ProcessTargets, verbose:
let shared_data = web::Data::new(AppState {
config: cfg,
targets,
verbose,
});
// Scheduler
@@ -117,7 +116,7 @@ async fn start_scheduler(expression: &String, data: Data<AppState>) -> ! {
if let Some(datetime) = upcoming.next() {
if datetime.timestamp() <= local.timestamp() {
playlist_processor::process_sources((&data.config).clone(), &data.targets, data.verbose);
playlist_processor::process_sources((&data.config).clone(), &data.targets);
}
}
}
src/config.rs
+23 -30
View File
@@ -1,13 +1,14 @@
use std::collections::{HashMap, HashSet};
use path_absolutize::*;
use enum_iterator::Sequence;
use log::{debug, error};
use crate::filter::{Filter, get_filter, MockValueProcessor, PatternTemplate, prepare_templates, ValueProvider};
use crate::mapping::Mappings;
use crate::mapping::Mapping;
use crate::model_config::{ItemField, ProcessingOrder, SortOrder, TargetType, default_as_zero, default_as_false, default_as_true};
use crate::user::{User};
use crate::utils;
use crate::{exit, utils};
use crate::utils::get_working_path;
fn default_as_frm() -> ProcessingOrder { ProcessingOrder::FRM }
@@ -53,8 +54,7 @@ impl ConfigSortChannel {
pub(crate) fn prepare(&mut self) {
let re = regex::Regex::new(&self.group_pattern);
if re.is_err() {
println!("cant parse regex: {}", &self.group_pattern);
std::process::exit(1);
exit!("cant parse regex: {}", &self.group_pattern);
}
self.re = Some(re.unwrap());
}
@@ -89,8 +89,7 @@ impl ConfigRename {
pub fn prepare(&mut self) {
let re = regex::Regex::new(&self.pattern);
if re.is_err() {
println!("cant parse regex: {}", &self.pattern);
std::process::exit(1);
exit!("cant parse regex: {}", &self.pattern);
}
self.re = Some(re.unwrap());
}
@@ -135,10 +134,10 @@ pub(crate) struct ConfigTarget {
}
impl ConfigTarget {
pub fn prepare(&mut self, id: u16, templates: Option<&Vec<PatternTemplate>>, verbose: bool) {
pub fn prepare(&mut self, id: u16, templates: Option<&Vec<PatternTemplate>>) {
self.id = id;
let fltr = get_filter(&self.filter, templates, verbose);
if verbose { println!("Filter: {}", fltr) }
let fltr = get_filter(&self.filter, templates);
debug!("Filter: {}", fltr);
self._filter = Some(fltr);
if let Some(renames) = self.rename.as_mut() {
renames.iter_mut().for_each(|r| r.prepare());
@@ -147,9 +146,9 @@ impl ConfigTarget {
sort.prepare();
}
}
pub fn filter(&self, provider: &ValueProvider, verbose: bool) -> bool {
pub fn filter(&self, provider: &ValueProvider) -> bool {
let mut processor = MockValueProcessor {};
return self._filter.as_ref().unwrap().filter(provider, &mut processor, verbose);
return self._filter.as_ref().unwrap().filter(provider, &mut processor);
}
}
@@ -203,8 +202,7 @@ impl ConfigInput {
pub fn prepare(&mut self, id: u16) {
self.id = id;
if self.url.trim().is_empty() {
println!("url for input is mandatory");
std::process::exit(1);
exit!("url for input is mandatory");
}
if let Some(user_name) = &self.username {
if user_name.trim().is_empty() {
@@ -219,13 +217,12 @@ impl ConfigInput {
match self.input_type {
InputType::M3u => {
if self.username.is_none() || self.password.is_none() {
println!("for input type m3u: username and password are ignored")
error!("for input type m3u: username and password are ignored")
}
}
InputType::Xtream => {
if self.username.is_none() || self.password.is_none() {
println!("for input type xtream: username and password are mandatory");
std::process::exit(1);
exit!("for input type xtream: username and password are mandatory");
}
}
}
@@ -284,12 +281,10 @@ impl Config {
let is_m3u = matches!(source.input.input_type, InputType::M3u);
for target in &mut source.targets {
if is_m3u && target.filename.is_none() {
println!("filename is required for m3u type: {}", target.name);
std::process::exit(1);
exit!("filename is required for m3u type: {}", target.name);
}
if !is_m3u && target.filename.is_none() && !target.publish {
println!("filename or publish is required for xtream type: {}", target.name);
std::process::exit(1);
exit!("filename or publish is required for xtream type: {}", target.name);
}
if let Some(mapping_ids) = &target.mapping {
@@ -307,11 +302,11 @@ impl Config {
}
}
pub fn prepare(&mut self, verbose: bool) {
pub fn prepare(&mut self) {
self.working_dir = get_working_path(&self.working_dir);
self.api.prepare();
self.prepare_api_web_root();
if let Some(templates) = &mut self.templates { self.templates = Some(prepare_templates(templates, verbose)) };
if let Some(templates) = &mut self.templates { self.templates = Some(prepare_templates(templates)) };
// prepare sources and set id's
let mut target_names_check = HashSet::<String>::new();
let default_target_name = default_as_default();
@@ -325,16 +320,15 @@ impl Config {
let target_name = target.name.clone();
if !default_target_name.eq_ignore_ascii_case(target_name.as_str()) {
if target_names_check.contains(target_name.as_str()) {
println!("target names should be unique: {}", target_name);
std::process::exit(1);
exit!("target names should be unique: {}", target_name);
} else {
target_names_check.insert(target_name);
}
}
// prepare templaes
match &self.templates {
Some(templ) => target.prepare(target_index, Some(templ), verbose),
_ => target.prepare(target_index, None, verbose)
Some(templ) => target.prepare(target_index, Some(templ)),
_ => target.prepare(target_index, None)
}
target_index += 1;
}
@@ -359,11 +353,11 @@ impl Config {
match wrpb2.absolutize() {
Ok(os) => self.api.web_root = String::from(os.to_str().unwrap()),
Err(e) => {
println!("failed to absolutize web_root {:?}", e);
error!("failed to absolutize web_root {:?}", e);
}
}
// } else {
// println!("web_root directory does not exists {:?}", wrpb2)
// error!("web_root directory does not exists {:?}", wrpb2)
}
}
}
@@ -404,11 +398,10 @@ pub(crate) fn validate_targets(target_args: &Option<Vec<String>>, sources: &Vec<
let missing_targets: Vec<String> = check_targets.iter().filter(|&(_, v)| *v == 0).map(|(k, _)| k.to_string()).collect();
if !missing_targets.is_empty() {
println!("No target found for {}", missing_targets.join(", "));
std::process::exit(1);
exit!("No target found for {}", missing_targets.join(", "));
}
let processing_targets: Vec<String> = check_targets.iter().filter(|&(_, v)| *v != 0).map(|(k, _)| k.to_string()).collect();
println!("Processing targets {}", processing_targets.join(", "));
debug!("Processing targets {}", processing_targets.join(", "));
} else {
enabled = false;
}
src/download.rs
+14 -15
View File
@@ -1,10 +1,11 @@
use std::path::PathBuf;
use std::sync::atomic::AtomicI32;
use log::debug;
use crate::{m3u_parser, utils, xtream_parser};
use crate::config::{Config, ConfigInput};
use crate::model_m3u::{PlaylistGroup, XtreamCluster};
fn prepare_file_path(input: &ConfigInput, working_dir: &String, action: &str, verbose: bool) -> Option<PathBuf> {
fn prepare_file_path(input: &ConfigInput, working_dir: &String, action: &str) -> Option<PathBuf> {
let persist_file: Option<PathBuf> =
match &input.persist {
Some(persist_path) => utils::prepare_persist_path(persist_path.as_str(), action),
@@ -12,22 +13,20 @@ fn prepare_file_path(input: &ConfigInput, working_dir: &String, action: &str, ve
};
if persist_file.is_some() {
let file_path = utils::get_file_path(working_dir, persist_file);
if verbose {
println!("persist to file: {:?}", match &file_path {
Some(fp) => fp.display().to_string(),
_ => "".to_string()
});
}
debug!("persist to file: {:?}", match &file_path {
Some(fp) => fp.display().to_string(),
_ => "".to_string()
});
file_path
} else {
None
}
}
pub(crate) fn get_m3u_playlist(cfg: &Config, input: &ConfigInput, working_dir: &String, verbose: bool) -> Option<Vec<PlaylistGroup>> {
pub(crate) fn get_m3u_playlist(cfg: &Config, input: &ConfigInput, working_dir: &String) -> Option<Vec<PlaylistGroup>> {
let url = input.url.as_str();
let file_path = prepare_file_path(input, working_dir, "", verbose);
let lines: Option<Vec<String>> = utils::get_input_content(working_dir, url, file_path, verbose);
let file_path = prepare_file_path(input, working_dir, "");
let lines: Option<Vec<String>> = utils::get_input_content(working_dir, url, file_path);
lines.map(|l| m3u_parser::parse_m3u(cfg, &l))
}
@@ -153,7 +152,7 @@ const ACTIONS: [(XtreamCluster, &str, &str); 3] = [
(XtreamCluster::VIDEO, "get_vod_categories", "get_vod_streams"),
(XtreamCluster::SERIES, "get_series_categories", "get_series")];
pub(crate) fn get_xtream_playlist(input: &ConfigInput, working_dir: &String, verbose: bool) -> Option<Vec<PlaylistGroup>> {
pub(crate) fn get_xtream_playlist(input: &ConfigInput, working_dir: &String) -> Option<Vec<PlaylistGroup>> {
let mut playlist: Vec<PlaylistGroup> = Vec::new();
let username = input.username.as_ref().unwrap().clone();
let password = input.password.as_ref().unwrap().clone();
@@ -164,11 +163,11 @@ pub(crate) fn get_xtream_playlist(input: &ConfigInput, working_dir: &String, ver
for (xtream_cluster, category, stream) in &ACTIONS {
let category_url = format!("{}&action={}", base_url, category);
let stream_url = format!("{}&action={}", base_url, stream);
let category_file_path = prepare_file_path(input, working_dir, format!("{}_", category).as_str(), verbose);
let stream_file_path = prepare_file_path(input, working_dir, format!("{}_", stream).as_str(), verbose);
let category_file_path = prepare_file_path(input, working_dir, format!("{}_", category).as_str());
let stream_file_path = prepare_file_path(input, working_dir, format!("{}_", stream).as_str());
let category_content: Option<serde_json::Value> = utils::get_input_json_content(input, &category_url, category_file_path, verbose);
let stream_content: Option<serde_json::Value> = utils::get_input_json_content(input, &stream_url, stream_file_path, verbose);
let category_content: Option<serde_json::Value> = utils::get_input_json_content(input, &category_url, category_file_path);
let stream_content: Option<serde_json::Value> = utils::get_input_json_content(input, &stream_url, stream_file_path);
let mut sub_playlist = xtream_parser::parse_xtream(&category_id_cnt, xtream_cluster, category_content, stream_content, &stream_base_url);
while let Some(group) = sub_playlist.pop() {
playlist.push(group);
src/filter.rs
+29 -34
View File
@@ -1,12 +1,14 @@
use std::cell::RefCell;
use enum_iterator::all;
use std::collections::{HashMap};
use log::{debug, error};
use pest::iterators::Pair;
use pest::Parser;
use petgraph::algo::toposort;
use crate::model_m3u::PlaylistItem;
use crate::model_config::ItemField;
use petgraph::graph::DiGraph;
use crate::exit;
pub(crate) fn get_field_value(pli: &PlaylistItem, field: &ItemField) -> String {
@@ -42,13 +44,13 @@ impl<'a> ValueProvider<'a> {
}
pub(crate) trait ValueProcessor {
fn process(&mut self, field: &ItemField, value: &str, rewc: &RegexWithCaptures, verbose: bool) -> bool;
fn process(&mut self, field: &ItemField, value: &str, rewc: &RegexWithCaptures) -> bool;
}
pub(crate) struct MockValueProcessor {}
impl ValueProcessor for MockValueProcessor {
fn process(&mut self, _: &ItemField, _: &str, _: &RegexWithCaptures, _: bool) -> bool {
fn process(&mut self, _: &ItemField, _: &str, _: &RegexWithCaptures) -> bool {
return false;
}
}
@@ -102,31 +104,31 @@ pub(crate) enum Filter {
}
impl Filter {
pub fn filter(&self, provider: &ValueProvider, processor: &mut dyn ValueProcessor, verbose: bool) -> bool {
pub fn filter(&self, provider: &ValueProvider, processor: &mut dyn ValueProcessor) -> bool {
match self {
Filter::Comparison(field, rewc) => {
let value = provider.call(field);
let is_match = rewc.re.is_match(value.as_str());
if is_match {
if verbose { println!("Match found: {:?} {} => {}={}", &rewc, &rewc.restr, &field, &value) }
processor.process(field, &value, rewc, verbose);
debug!("Match found: {:?} {} => {}={}", &rewc, &rewc.restr, &field, &value);
processor.process(field, &value, rewc);
}
is_match
}
Filter::Group(expr) => {
expr.filter(provider, processor, verbose)
expr.filter(provider, processor)
}
Filter::UnaryExpression(op, expr) => {
match op {
UnaryOperator::NOT => !expr.filter(provider, processor, verbose),
UnaryOperator::NOT => !expr.filter(provider, processor),
}
}
Filter::BinaryExpression(left, op, right) => {
match op {
BinaryOperator::AND => left.filter(provider, processor, verbose)
&& right.filter(provider, processor, verbose),
BinaryOperator::OR => left.filter(provider, processor, verbose)
|| right.filter(provider, processor, verbose),
BinaryOperator::AND => left.filter(provider, processor)
&& right.filter(provider, processor),
BinaryOperator::OR => left.filter(provider, processor)
|| right.filter(provider, processor),
}
}
}
@@ -155,13 +157,6 @@ impl std::fmt::Display for Filter {
}
}
macro_rules! exit {
($($arg:tt)*) => {{
println!($($arg)*);
std::process::exit(1);
}};
}
fn get_parser_item_field(expr: Pair<Rule>) -> ItemField {
if expr.as_rule() == Rule::field {
let field_text = expr.as_str();
@@ -174,7 +169,7 @@ fn get_parser_item_field(expr: Pair<Rule>) -> ItemField {
exit!("unknown field: {}", expr.as_str());
}
fn get_parser_regexp(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, verbose: bool) -> RegexWithCaptures {
fn get_parser_regexp(expr: Pair<Rule>, templates: &Vec<PatternTemplate>) -> RegexWithCaptures {
if expr.as_rule() == Rule::regexp {
let mut parsed_text = String::from(expr.as_str());
parsed_text.pop();
@@ -190,7 +185,7 @@ fn get_parser_regexp(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, verbose
let regexp = re.unwrap();
let captures = regexp.capture_names()
.flatten().map(String::from).filter(|x| !x.is_empty()).collect::<Vec<String>>();
if verbose { println!("Created regex: {} with captures: [{}]", regstr, captures.join(", ")) }
debug!("Created regex: {} with captures: [{}]", regstr, captures.join(", "));
return RegexWithCaptures {
restr: regstr,
re: regexp,
@@ -200,10 +195,10 @@ fn get_parser_regexp(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, verbose
exit!("unknown field: {}", expr.as_str());
}
fn get_parser_comparison(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, verbose: bool) -> Filter {
fn get_parser_comparison(expr: Pair<Rule>, templates: &Vec<PatternTemplate>) -> Filter {
let mut expr_inner = expr.into_inner();
let field = get_parser_item_field(expr_inner.next().unwrap());
let regexp = get_parser_regexp(expr_inner.next().unwrap(), templates, verbose);
let regexp = get_parser_regexp(expr_inner.next().unwrap(), templates);
Filter::Comparison(field, regexp)
}
@@ -229,7 +224,7 @@ macro_rules! handle_expr {
}
}
fn get_parser_expression(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, verbose: bool) -> Filter {
fn get_parser_expression(expr: Pair<Rule>, templates: &Vec<PatternTemplate>) -> Filter {
let mut stmts = Vec::new();
let pairs = expr.into_inner();
let mut bop: Option<BinaryOperator> = None;
@@ -238,13 +233,13 @@ fn get_parser_expression(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, ver
for pair in pairs {
match pair.as_rule() {
Rule::comparison => {
handle_expr!(bop, uop, stmts, get_parser_comparison(pair, templates, verbose));
handle_expr!(bop, uop, stmts, get_parser_comparison(pair, templates));
}
Rule::expr => {
handle_expr!(bop, uop, stmts, get_parser_expression(pair, templates, verbose));
handle_expr!(bop, uop, stmts, get_parser_expression(pair, templates));
}
Rule::expr_group => {
handle_expr!(bop, uop, stmts, Filter::Group(Box::new(get_parser_expression(pair.into_inner().next().unwrap(), templates, verbose))));
handle_expr!(bop, uop, stmts, Filter::Group(Box::new(get_parser_expression(pair.into_inner().next().unwrap(), templates))));
}
Rule::not => {
uop = Some(UnaryOperator::NOT);
@@ -253,7 +248,7 @@ fn get_parser_expression(expr: Pair<Rule>, templates: &Vec<PatternTemplate>, ver
bop = Some(get_parser_binary_op(pair.into_inner().next().unwrap()));
}
_ => {
println!("did not expect rule: {:?}", pair)
error!("did not expect rule: {:?}", pair)
}
}
}
@@ -273,7 +268,7 @@ fn get_parser_binary_op(expr: Pair<Rule>) -> BinaryOperator {
}
}
pub(crate) fn get_filter(filter_text: &str, templates: Option<&Vec<PatternTemplate>>, verbose: bool) -> Filter {
pub(crate) fn get_filter(filter_text: &str, templates: Option<&Vec<PatternTemplate>>) -> Filter {
let empty_list = Vec::new();
let template_list: &Vec<PatternTemplate> = templates.unwrap_or(&empty_list);
let mut source = String::from(filter_text);
@@ -291,7 +286,7 @@ pub(crate) fn get_filter(filter_text: &str, templates: Option<&Vec<PatternTempla
for expr in pair.into_inner() {
match expr.as_rule() {
Rule::expr => {
let expr = get_parser_expression(expr, template_list, verbose);
let expr = get_parser_expression(expr, template_list);
match &op {
Some(binop) => {
result = Some(Filter::BinaryExpression(Box::new(result.unwrap()), binop.clone(), Box::new(expr)));
@@ -304,7 +299,7 @@ pub(crate) fn get_filter(filter_text: &str, templates: Option<&Vec<PatternTempla
op = Some(get_parser_binary_op(expr.into_inner().next().unwrap()));
}
_ => {
println!("unknown expression {:?}", expr);
error!("unknown expression {:?}", expr);
//exit(format!("unknown stmt inner: {}", expr.as_str()).as_str());
}
}
@@ -364,13 +359,13 @@ fn build_dependency_graph(templates: &Vec<PatternTemplate>) -> GraphDependency {
.map(|scc| scc.iter().map(|&i| node_names.get(&i.index()).unwrap().clone()).collect())
.collect();
for cyclic in &cycles {
println!("Cyclic template dependencies detected [{}]", cyclic.join(" <-> "))
error!("Cyclic template dependencies detected [{}]", cyclic.join(" <-> "))
}
(graph, node_names, node_deps, !cycles.is_empty())
}
pub(crate) fn prepare_templates(templates: &Vec<PatternTemplate>, verbose: bool) -> Vec<PatternTemplate> {
pub(crate) fn prepare_templates(templates: &Vec<PatternTemplate>) -> Vec<PatternTemplate> {
let mut result: Vec<PatternTemplate> = templates.to_vec();
let (graph, node_map, node_deps, cyclic) = build_dependency_graph(templates);
if cyclic {
@@ -385,7 +380,7 @@ pub(crate) fn prepare_templates(templates: &Vec<PatternTemplate>, verbose: bool)
if graph.edges_directed(*node, petgraph::Incoming).count() > 0 {
let node_name = node_map.get(&node.index()).unwrap();
if let Some(deps) = node_deps.get(node_name) {
if verbose { println!("template {} depends on [{}]", node_name, deps.join(", ")) };
debug!("template {} depends on [{}]", node_name, deps.join(", "));
let mut node_template = dep_value_map.get(node_name).unwrap().clone();
for dep_name in deps {
let dep_template = dep_value_map.get(dep_name).unwrap().clone();
@@ -400,6 +395,6 @@ pub(crate) fn prepare_templates(templates: &Vec<PatternTemplate>, verbose: bool)
}
}
}
if verbose { println!("{:#?}", result); }
debug!("{:#?}", result);
result
}
src/m3u_repository.rs
+4 -3
View File
@@ -4,13 +4,14 @@ use crate::utils;
use std::io::Write;
use chrono::Datelike;
use crate::model_config::TargetType;
use log::{error};
macro_rules! open_file {
($path:expr) => {{
match std::fs::File::create($path) {
Ok(file) => file,
Err(e) => {
println!("cant create file: {:?}", $path);
error!("cant create file: {:?}", $path);
return Err(e);
}
}
@@ -123,14 +124,14 @@ fn write_strm_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &mut V
let _ = std::fs::remove_dir_all(&path);
}
if let Err(e) = std::fs::create_dir_all(&path) {
println!("cant create directory: {:?}", &path);
error!("cant create directory: {:?}", &path);
return Err(e);
};
for pg in new_playlist {
for pli in &pg.channels {
let dir_path = path.join(sanitize_for_filename(&pli.header.borrow().group, underscore_whitespace));
if let Err(e) = std::fs::create_dir_all(&dir_path) {
println!("cant create directory: {:?}", &path);
error!("cant create directory: {:?}", &path);
return Err(e);
};
let mut file_name = sanitize_for_filename(&pli.header.borrow().title, underscore_whitespace);
src/main.rs
+46 -31
View File
@@ -1,6 +1,9 @@
extern crate pest;
#[macro_use]
extern crate pest_derive;
extern crate env_logger;
use env_logger::{Builder};
use log::{debug, error, info, LevelFilter, warn};
use clap::Parser;
use crate::config::{Config, validate_targets};
@@ -52,83 +55,95 @@ struct Args {
#[arg(short, long, default_value_t = false, default_missing_value = "true")]
server: bool,
/// Print more info
#[arg(short, long, default_value_t = false)]
verbose: bool,
/// log level
#[arg(short, long = "log-level", default_missing_value = "info")]
log_level: Option<String>,
}
fn main() {
let args = Args::parse();
let verbose = args.verbose;
init_logger(&args.log_level.unwrap_or("info".to_string()));
let default_config_path = utils::get_default_config_path();
let config_file: String = args.config.unwrap_or(default_config_path);
let mut cfg = read_config(config_file.as_str(), verbose);
let mut cfg = read_config(config_file.as_str());
let targets = validate_targets(&args.target, &cfg.sources);
if verbose { println!("working dir: {:?}", &cfg.working_dir); }
read_mappings(args.mapping, &mut cfg, verbose);
info!("working dir: {:?}", &cfg.working_dir);
read_mappings(args.mapping, &mut cfg);
if args.server {
read_users(args.user, &mut cfg, verbose);
if verbose { println!("web_root: {}", &cfg.api.web_root); }
println!("server running: http://{}:{}", &cfg.api.host, &cfg.api.port);
match api::start_server(cfg, targets, verbose) {
read_users(args.user, &mut cfg);
debug!("web_root: {}", &cfg.api.web_root);
info!("server running: http://{}:{}", &cfg.api.host, &cfg.api.port);
match api::start_server(cfg, targets) {
Ok(_) => {}
Err(e) => {
println!("cant start server: {}", e);
std::process::exit(1);
exit!("cant start server: {}", e);
}
};
} else {
playlist_processor::process_sources(cfg, &targets, verbose);
playlist_processor::process_sources(cfg, &targets);
}
}
fn read_mappings(args_mapping: Option<String>, cfg: &mut Config, verbose: bool) {
fn init_logger(log_level: &String) {
let mut log_builder = Builder::new();
// Set the log level based on the parsed value
match log_level.to_lowercase().as_str() {
"trace" => log_builder.filter_level(LevelFilter::Trace),
"debug" => log_builder.filter_level(LevelFilter::Debug),
"info" => log_builder.filter_level(LevelFilter::Info),
"warn" => log_builder.filter_level(LevelFilter::Warn),
"error" => log_builder.filter_level(LevelFilter::Error),
_ => log_builder.filter_level(LevelFilter::Info),
};
log_builder.init();
}
fn read_mappings(args_mapping: Option<String>, cfg: &mut Config) {
let mappings_file: String = args_mapping.unwrap_or(utils::get_default_mappings_path());
let mappings = read_mapping(mappings_file.as_str(), verbose);
if verbose && mappings.is_none() { println!("no mapping loaded"); }
let mappings = read_mapping(mappings_file.as_str());
if mappings.is_none() { debug!("no mapping loaded"); }
cfg.set_mappings(mappings);
}
fn read_users(args_user: Option<String>, cfg: &mut Config, verbose: bool) {
fn read_users(args_user: Option<String>, cfg: &mut Config) {
let user_file: String = args_user.unwrap_or(utils::get_default_user_path());
let user = read_user(user_file.as_str(), verbose);
let user = read_user(user_file.as_str());
if user.is_none() {
println!("cant read user file: {}", user_file.as_str());
//std::process::exit(1);
warn!("cant read user file: {}", user_file.as_str());
}
cfg.set_user(user);
}
fn read_config(config_file: &str, verbose: bool) -> Config {
fn read_config(config_file: &str) -> Config {
let mut cfg: Config = match serde_yaml::from_reader(utils::open_file(&std::path::PathBuf::from(config_file), true).unwrap()) {
Ok(result) => result,
Err(e) => {
println!("cant read config file: {}", e);
std::process::exit(1);
exit!("cant read config file: {}", e);
}
};
cfg.prepare(verbose);
cfg.prepare();
cfg
}
fn read_mapping(mapping_file: &str, verbose: bool) -> Option<Mappings> {
fn read_mapping(mapping_file: &str) -> Option<Mappings> {
match utils::open_file(&std::path::PathBuf::from(mapping_file), false) {
Some(file) => {
let mapping: Result<Mappings, _> = serde_yaml::from_reader(file);
match mapping {
Ok(mut result) => {
result.prepare(verbose);
result.prepare();
Some(result)
}
Err(err) => {
println!("cant read mapping file: {}", err);
error!("cant read mapping file: {}", err);
None
}
}
@@ -137,17 +152,17 @@ fn read_mapping(mapping_file: &str, verbose: bool) -> Option<Mappings> {
}
}
fn read_user(user_file: &str, verbose: bool) -> Option<User> {
fn read_user(user_file: &str) -> Option<User> {
match utils::open_file(&std::path::PathBuf::from(user_file), false) {
Some(file) => {
let mapping: Result<User, _> = serde_yaml::from_reader(file);
match mapping {
Ok(result) => {
result.prepare(verbose);
result.prepare();
Some(result)
}
Err(err) => {
println!("cant read user file: {}", err);
error!("cant read user file: {}", err);
None
}
}
src/mapping.rs
+32 -31
View File
@@ -1,5 +1,6 @@
use std::cell::RefCell;
use std::collections::HashMap;
use log::{debug, error};
use regex::Regex;
use crate::filter::{Filter, get_filter, PatternTemplate, prepare_templates, RegexWithCaptures, ValueProcessor};
use crate::model_m3u::{FieldAccessor, PlaylistItem};
@@ -42,11 +43,11 @@ pub(crate) struct Mapper {
}
impl Mapper {
pub fn prepare(&mut self, templates: Option<&Vec<PatternTemplate>>, tags: Option<&Vec<MappingTag>>, verbose: bool) {
self._pattern = Some(get_filter(&self.pattern, templates, verbose));
pub fn prepare(&mut self, templates: Option<&Vec<PatternTemplate>>, tags: Option<&Vec<MappingTag>>) {
self._pattern = Some(get_filter(&self.pattern, templates));
match &self.filter {
Some(flt) => {
self._filter = Some(get_filter(flt, templates, verbose));
self._filter = Some(get_filter(flt, templates));
},
_ => self._filter = None
}
@@ -68,25 +69,25 @@ impl MappingValueProcessor<'_> {
self.pli.borrow().header.borrow().get_field(key).map(String::from)
}
fn set_property(&mut self, key: &str, value: &String, verbose: bool) {
fn set_property(&mut self, key: &str, value: &String) {
if !self.pli.borrow().header.borrow_mut().set_field(key, value) {
println!("Cant set unknown field {} to {}", key, value);
error!("Cant set unknown field {} to {}", key, value);
}
if verbose { println!("Property {} set to {}", key, value) }
debug!("Property {} set to {}", key, value);
}
fn apply_attributes(&mut self, verbose: bool) {
fn apply_attributes(&mut self) {
let mapper = self.mapper.borrow();
let attributes = &mapper.attributes;
drop(mapper);
for (key, value) in attributes {
if valid_property!(key.as_str(), MAPPER_ATTRIBUTE_FIELDS) {
self.set_property(key, value, verbose);
self.set_property(key, value);
}
}
}
fn apply_tags(&mut self, value: &String, captures: &HashMap<&String, &str>, verbose: bool) -> Option<String> {
fn apply_tags(&mut self, value: &String, captures: &HashMap<&String, &str>) -> Option<String> {
let mut new_value = String::from(value);
let tag_captures = self.mapper.borrow()._tagre.as_ref().unwrap().captures_iter(value)
.filter(|caps| caps.len() > 1)
@@ -103,7 +104,7 @@ impl MappingValueProcessor<'_> {
match captures.get(&cap) {
Some(cap_value) => captured_tag_values.push(cap_value),
_ => {
if verbose { println!("Cant find any tag match for {}", tag_capture) }
debug!("Cant find any tag match for {}", tag_capture);
return None;
}
}
@@ -125,40 +126,40 @@ impl MappingValueProcessor<'_> {
Some(new_value)
}
fn apply_suffix(&mut self, captures: &HashMap<&String, &str>, verbose: bool) {
fn apply_suffix(&mut self, captures: &HashMap<&String, &str>) {
let mapper = self.mapper.borrow();
let suffix = &mapper.suffix;
drop(mapper);
for (key, value) in suffix {
if valid_property!(key.as_str(), AFFIX_FIELDS) {
if let Some(suffix) = self.apply_tags(value, captures, verbose) {
if let Some(suffix) = self.apply_tags(value, captures) {
if let Some(old_value) = self.get_property(key) {
let new_value = format!("{}{}", &old_value, suffix);
self.set_property(key, &new_value, verbose);
self.set_property(key, &new_value);
}
}
}
}
}
fn apply_prefix(&mut self, captures: &HashMap<&String, &str>, verbose: bool) {
fn apply_prefix(&mut self, captures: &HashMap<&String, &str>) {
let mapper = self.mapper.borrow();
let prefix = &mapper.prefix;
drop(mapper);
for (key, value) in prefix {
if valid_property!(key.as_str(), AFFIX_FIELDS) {
if let Some(prefix) = self.apply_tags(value, captures, verbose) {
if let Some(prefix) = self.apply_tags(value, captures) {
if let Some(old_value) = self.get_property(key) {
let new_value = format!("{}{}", prefix, &old_value);
self.set_property(key, &new_value, verbose);
self.set_property(key, &new_value);
}
}
}
}
}
fn apply_assignments(&mut self, verbose: bool) {
fn apply_assignments(&mut self) {
let mapper = self.mapper.borrow();
let assignments = &mapper.assignments;
drop(mapper);
@@ -166,7 +167,7 @@ impl MappingValueProcessor<'_> {
if valid_property!(key.as_str(), MAPPER_ATTRIBUTE_FIELDS) &&
valid_property!(value.as_str(), MAPPER_ATTRIBUTE_FIELDS) {
if let Some(prop_value) = self.get_property(value) {
self.set_property(key, &prop_value, verbose);
self.set_property(key, &prop_value);
}
}
}
@@ -174,7 +175,7 @@ impl MappingValueProcessor<'_> {
}
impl ValueProcessor for MappingValueProcessor<'_> {
fn process<'a>(&mut self, _: &ItemField, value: &str, rewc: &RegexWithCaptures, verbose: bool) -> bool {
fn process<'a>(&mut self, _: &ItemField, value: &str, rewc: &RegexWithCaptures) -> bool {
let mut captured_values = HashMap::new();
if !rewc.captures.is_empty() {
rewc.re.captures_iter(value)
@@ -187,15 +188,15 @@ impl ValueProcessor for MappingValueProcessor<'_> {
} else {
""
};
if verbose { println!("match {}: {}", capture_name, capture_value); }
debug!("match {}: {}", capture_name, capture_value);
captured_values.insert(capture_name, capture_value);
}
);
}
let _ = &MappingValueProcessor::<'_>::apply_attributes(self, verbose);
let _ = &MappingValueProcessor::<'_>::apply_suffix(self, &captured_values, verbose);
let _ = &MappingValueProcessor::<'_>::apply_prefix(self, &captured_values, verbose);
let _ = &MappingValueProcessor::<'_>::apply_assignments(self, verbose);
let _ = &MappingValueProcessor::<'_>::apply_attributes(self);
let _ = &MappingValueProcessor::<'_>::apply_suffix(self, &captured_values);
let _ = &MappingValueProcessor::<'_>::apply_prefix(self, &captured_values);
let _ = &MappingValueProcessor::<'_>::apply_assignments(self);
true
}
}
@@ -212,9 +213,9 @@ pub(crate) struct Mapping {
impl Mapping {
pub fn prepare(&mut self, templates: Option<&Vec<PatternTemplate>>,
tags: Option<&Vec<MappingTag>>, verbose: bool) {
tags: Option<&Vec<MappingTag>>) {
for mapper in &mut self.mapper {
mapper.prepare(templates, tags, verbose);
mapper.prepare(templates, tags);
}
}
}
@@ -227,8 +228,8 @@ pub(crate) struct MappingDefinition {
}
impl MappingDefinition {
pub fn prepare(&mut self, verbose: bool) {
if let Some(templates) = &mut self.templates { self.templates = Some(prepare_templates(templates, verbose)) };
pub fn prepare(&mut self) {
if let Some(templates) = &mut self.templates { self.templates = Some(prepare_templates(templates)) };
for mapping in &mut self.mapping {
let template_list = match &self.templates {
Some(templ) => Some(templ),
@@ -238,7 +239,7 @@ impl MappingDefinition {
Some(t) => Some(t),
_ => None
};
mapping.prepare(template_list, tag_list, verbose);
mapping.prepare(template_list, tag_list);
}
}
}
@@ -249,8 +250,8 @@ pub(crate) struct Mappings {
}
impl Mappings {
pub fn prepare(&mut self, verbose: bool) {
self.mappings.prepare(verbose);
pub fn prepare(&mut self) {
self.mappings.prepare();
}
pub fn get_mapping(&self, mapping_id: &String) -> Option<Mapping> {
src/messaging.rs
+1 -1
View File
@@ -1,3 +1,3 @@
pub(crate) fn send_message(msg: &str) {
pub(crate) fn send_message(_msg: &str) {
}
src/model_api.rs
-1
View File
@@ -5,7 +5,6 @@ use crate::config::{Config, ProcessTargets};
pub(crate) struct AppState {
pub config: Config,
pub targets: ProcessTargets,
pub verbose: bool,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
src/playlist_processor.rs
+42 -42
View File
@@ -3,6 +3,7 @@ extern crate unidecode;
use std::cell::RefCell;
use std::sync::{Arc};
use std::thread;
use log::{debug, error};
use config::ConfigTarget;
use unidecode::unidecode;
use crate::{config, Config, valid_property};
@@ -16,18 +17,18 @@ use crate::model_config::{ItemField, AFFIX_FIELDS, ProcessingOrder};
use crate::download::{get_m3u_playlist, get_xtream_playlist};
use crate::repository::save_playlist;
fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbose: bool) -> Option<Vec<PlaylistGroup>> {
if verbose { println!("Filtering {} groups", playlist.len()) }
fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option<Vec<PlaylistGroup>> {
debug!("Filtering {} groups", playlist.len());
let mut new_playlist = Vec::new();
playlist.iter_mut().for_each(|pg| {
if verbose { println!("Filtering group {} with {} items", pg.title, pg.channels.len()) }
debug!("Filtering group {} with {} items", pg.title, pg.channels.len());
let mut channels = Vec::new();
pg.channels.iter_mut().for_each(|pli| {
if is_valid(pli, target, verbose) {
if is_valid(pli, target) {
channels.push(pli.clone());
}
});
if verbose { println!("Filtered group {} has now {} items", pg.title, channels.len()) }
debug!("Filtered group {} has now {} items", pg.title, channels.len());
if !channels.is_empty() {
new_playlist.push(PlaylistGroup {
id: pg.id,
@@ -40,7 +41,7 @@ fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbos
Some(new_playlist)
}
fn apply_affixes(playlist: &mut [PlaylistGroup], input: &ConfigInput, verbose: bool) {
fn apply_affixes(playlist: &mut [PlaylistGroup], input: &ConfigInput) {
if input.suffix.is_some() || input.prefix.is_some() {
let validate_affix = |a: &Option<InputAffix>| match a {
Some(affix) => {
@@ -69,14 +70,14 @@ fn apply_affixes(playlist: &mut [PlaylistGroup], input: &ConfigInput, verbose: b
if apply_suffix {
if let Some(suffix) = &input.suffix {
let value = get_affix_applied_value(&mut channel.header.borrow_mut(), suffix, false);
if verbose { println!("Applying input suffix: {}={}", &suffix.field, &value) }
debug!("Applying input suffix: {}={}", &suffix.field, &value);
channel.header.borrow_mut().set_field(&suffix.field, value.as_str());
}
}
if apply_prefix {
if let Some(prefix) = &input.prefix {
let value = get_affix_applied_value(&mut channel.header.borrow_mut(), prefix, true);
if verbose { println!("Applying input prefix: {}={}", &prefix.field, &value) }
debug!("Applying input prefix: {}={}", &prefix.field, &value);
channel.header.borrow_mut().set_field(&prefix.field, value.as_str());
}
}
@@ -126,19 +127,19 @@ fn sort_playlist(target: &ConfigTarget, new_playlist: &mut [PlaylistGroup]) {
}
fn is_valid(pli: &mut PlaylistItem, target: &ConfigTarget, verbose: bool) -> bool {
fn is_valid(pli: &mut PlaylistItem, target: &ConfigTarget) -> bool {
let provider = ValueProvider { pli: RefCell::new(pli) };
target.filter(&provider, verbose)
target.filter(&provider)
}
fn exec_rename(pli: &mut PlaylistItem, rename: &Option<Vec<config::ConfigRename>>, verbose: bool) {
fn exec_rename(pli: &mut PlaylistItem, rename: &Option<Vec<config::ConfigRename>>) {
if let Some(renames) = rename {
if !renames.is_empty() {
let result = pli;
for r in renames {
let value = get_field_value(result, &r.field);
let cap = r.re.as_ref().unwrap().replace_all(value.as_str(), &r.new_name);
if verbose { println!("Renamed {}={} to {}", &r.field, value, cap) }
debug!("Renamed {}={} to {}", &r.field, value, cap);
let value = cap.into_owned();
set_field_value(result, &r.field, value);
}
@@ -146,7 +147,7 @@ fn exec_rename(pli: &mut PlaylistItem, rename: &Option<Vec<config::ConfigRename>
}
}
fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbose: bool) -> Option<Vec<PlaylistGroup>> {
fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option<Vec<PlaylistGroup>> {
match &target.rename {
Some(renames) => {
if !renames.is_empty() {
@@ -156,12 +157,12 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbos
for r in renames {
if let ItemField::Group = r.field {
let cap = r.re.as_ref().unwrap().replace_all(&grp.title, &r.new_name);
if verbose { println!("Renamed group {} to {}", &grp.title, cap); }
debug!("Renamed group {} to {}", &grp.title, cap);
grp.title = cap.into_owned();
}
}
grp.channels.iter_mut().for_each(|pli| exec_rename(pli, &target.rename, verbose));
grp.channels.iter_mut().for_each(|pli| exec_rename(pli, &target.rename));
new_playlist.push(grp);
}
return Some(new_playlist);
@@ -173,21 +174,21 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbos
}
macro_rules! apply_pattern {
($pattern:expr, $provider:expr, $processor:expr, $verbose:expr) => {{
($pattern:expr, $provider:expr, $processor:expr) => {{
match $pattern {
Some(ptrn) => {
ptrn.filter($provider, $processor, $verbose);
ptrn.filter($provider, $processor);
},
_ => {}
};
}};
}
fn map_channel(channel: &PlaylistItem, mapping: &Mapping, verbose: bool) -> PlaylistItem {
fn map_channel(channel: &PlaylistItem, mapping: &Mapping) -> PlaylistItem {
if !mapping.mapper.is_empty() {
let header = channel.header.borrow();
let channel_name = if mapping.match_as_ascii { unidecode(&header.name) } else { String::from(&header.name) };
if verbose && mapping.match_as_ascii { println!("Decoded {} for matching to {}", &header.name, &channel_name) };
if mapping.match_as_ascii { debug!("Decoded {} for matching to {}", &header.name, &channel_name); };
drop(header);
let ref_chan = RefCell::new(channel);
let provider = ValueProvider { pli: ref_chan.clone() };
@@ -197,12 +198,12 @@ fn map_channel(channel: &PlaylistItem, mapping: &Mapping, verbose: bool) -> Play
let mut processor = MappingValueProcessor { pli: ref_chan.clone(), mapper: RefCell::new(m) };
match &m._filter {
Some(filter) => {
if filter.filter(&provider, &mut mock_processor, verbose) {
apply_pattern!(&m._pattern, &provider, &mut processor, verbose);
if filter.filter(&provider, &mut mock_processor) {
apply_pattern!(&m._pattern, &provider, &mut processor);
}
}
_ => {
apply_pattern!(&m._pattern, &provider, &mut processor, verbose);
apply_pattern!(&m._pattern, &provider, &mut processor);
}
};
}
@@ -210,14 +211,14 @@ fn map_channel(channel: &PlaylistItem, mapping: &Mapping, verbose: bool) -> Play
channel.clone()
}
fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbose: bool) -> Option<Vec<PlaylistGroup>> {
if verbose { println!("Mapping") }
fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option<Vec<PlaylistGroup>> {
debug!("Mapping");
if target._mapping.is_some() {
let new_playlist: Vec<PlaylistGroup> = playlist.iter().map(|playlist_group| {
let mut grp = playlist_group.clone();
let mappings = target._mapping.as_ref().unwrap();
mappings.iter().filter(|mapping| !mapping.mapper.is_empty()).for_each(|mapping|
grp.channels = grp.channels.iter_mut().map(|chan| map_channel(chan, mapping, verbose)).collect());
grp.channels = grp.channels.iter_mut().map(|chan| map_channel(chan, mapping)).collect());
grp
}).collect();
@@ -248,26 +249,26 @@ fn map_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbose:
}
}
fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<ProcessTargets>, verbose: bool) {
fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<ProcessTargets>) {
let source = cfg.sources.get(source_idx).unwrap();
let input = &source.input;
if input.enabled || (user_targets.enabled && user_targets.has_input(input.id)) {
let mut result = match input.input_type {
InputType::M3u => get_m3u_playlist(&cfg, input, &cfg.working_dir, verbose),
InputType::Xtream => get_xtream_playlist(input, &cfg.working_dir, verbose),
InputType::M3u => get_m3u_playlist(&cfg, input, &cfg.working_dir),
InputType::Xtream => get_xtream_playlist(input, &cfg.working_dir),
};
if let Some(playlist) = result.as_mut() {
if playlist.is_empty() {
if verbose { println!("Input file is empty") }
debug!("Input file is empty");
} else {
if verbose { println!("Input file has {} groups", playlist.len()) }
debug!("Input file has {} groups", playlist.len());
source.targets.iter().for_each(|target| {
let should_process = (!user_targets.enabled && target.enabled)
|| (user_targets.enabled && user_targets.has_target(target.id));
if should_process {
match process_playlist(playlist, input, target, &cfg, verbose) {
match process_playlist(playlist, input, target, &cfg) {
Ok(_) => (),
Err(e) => println!("Failed to write file: {}", e)
Err(e) => error!("Failed to write file: {}", e)
}
}
});
@@ -276,17 +277,17 @@ fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<Process
}
}
pub(crate) fn process_sources(cfg: Config, user_targets: &ProcessTargets, verbose: bool) {
pub(crate) fn process_sources(cfg: Config, user_targets: &ProcessTargets) {
let config = Arc::new(cfg);
let mut handle_list = vec![];
let thread_num = config.threads;
let process_parallel = thread_num > 1 && config.sources.len() > 1;
if verbose && process_parallel { println!("Using {} threads", thread_num) }
if process_parallel { debug!("Using {} threads", thread_num); }
for (index, _) in config.sources.iter().enumerate() {
let cfg_clone = config.clone();
let usr_targets = Arc::new(user_targets.clone());
let process = move || process_source(cfg_clone, index, usr_targets, verbose);
let process = move || process_source(cfg_clone, index, usr_targets);
if process_parallel {
let handles = &mut handle_list;
handles.push(thread::spawn(process));
@@ -305,12 +306,11 @@ pub(crate) fn process_sources(cfg: Config, user_targets: &ProcessTargets, verbos
}
type ProcessingPipe = Vec<fn(playlist: &mut [PlaylistGroup], target: &ConfigTarget, verbose: bool) -> Option<Vec<PlaylistGroup>>>;
type ProcessingPipe = Vec<fn(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Option<Vec<PlaylistGroup>>>;
pub(crate) fn process_playlist(playlist: &mut [PlaylistGroup],
input: &ConfigInput,
target: &ConfigTarget, cfg: &Config,
verbose: bool) -> Result<(), std::io::Error> {
target: &ConfigTarget, cfg: &Config) -> Result<(), std::io::Error> {
let pipe: ProcessingPipe =
match &target.processing_order {
ProcessingOrder::FRM => vec![filter_playlist, rename_playlist, map_playlist],
@@ -321,17 +321,17 @@ pub(crate) fn process_playlist(playlist: &mut [PlaylistGroup],
ProcessingOrder::MRF => vec![map_playlist, rename_playlist, filter_playlist]
};
if verbose { println!("Processing order is {}", &target.processing_order) }
debug!("Processing order is {}", &target.processing_order);
let mut new_playlist = playlist.to_owned();
for f in pipe {
let r = f(&mut new_playlist, target, verbose);
let r = f(&mut new_playlist, target);
if let Some(v) = r {
new_playlist = v;
}
}
apply_affixes(&mut new_playlist, input, verbose);
apply_affixes(&mut new_playlist, input);
sort_playlist(target, &mut new_playlist);
let publish = target.publish;
@@ -340,7 +340,7 @@ pub(crate) fn process_playlist(playlist: &mut [PlaylistGroup],
match &result {
Ok(..) => {},
Err(e) => {
println!("failed to write {:?}", e);
error!("failed to write {:?}", e);
}
}
if !publish {
src/repository.rs
+16 -11
View File
@@ -9,6 +9,7 @@ use crate::config::{Config, ConfigTarget};
use crate::messaging::send_message;
use crate::model_m3u::{PlaylistGroup, XtreamCluster};
use crate::utils;
use log::{error};
pub(crate) static COL_CAT_LIVE: &str = "cat_live";
pub(crate) static COL_CAT_SERIES: &str = "cat_series";
@@ -60,6 +61,10 @@ fn get_storage_path(cfg: &Config, target_name: &str) -> Option<PathBuf> {
utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(" ", "_"))))
}
fn get_collection_path(path: &PathBuf, collection: &str) -> PathBuf {
path.join(format!("{}.json", collection))
}
pub(crate) fn save_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut Vec<PlaylistGroup>) -> Result<(), std::io::Error> {
let mut failed = false;
if let Some(path) = get_storage_path(&cfg, &target.name) {
@@ -119,17 +124,17 @@ pub(crate) fn save_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut
}
for (col_path, data) in [
(&path.join(COL_CAT_LIVE), &cat_live_col),
(&path.join(COL_CAT_VOD), &cat_vod_col),
(&path.join(COL_CAT_SERIES), &cat_series_col),
(&path.join(COL_LIVE), &live_col),
(&path.join(COL_VOD), &vod_col),
(&path.join(COL_SERIES), &series_col)] {
match write_to_file(col_path, data) {
(get_collection_path(&path, COL_CAT_LIVE), &cat_live_col),
(get_collection_path(&path, COL_CAT_VOD), &cat_vod_col),
(get_collection_path(&path, COL_CAT_SERIES), &cat_series_col),
(get_collection_path(&path, COL_LIVE), &live_col),
(get_collection_path(&path, COL_VOD), &vod_col),
(get_collection_path(&path, COL_SERIES), &series_col)] {
match write_to_file(&col_path, data) {
Ok(()) => {}
Err(err) => {
failed = true;
println!("Persisting collection failed: {}: {}", &col_path.clone().into_os_string().into_string().unwrap(), err);
error!("Persisting collection failed: {}: {}", &col_path.clone().into_os_string().into_string().unwrap(), err);
}
}
}
@@ -145,10 +150,10 @@ pub(crate) fn save_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut
pub(crate) fn get_all(cfg: &Config, target_name: &str, collection_name: &str) -> Result<PathBuf, Error> {
if let Some(path) = get_storage_path(&cfg, target_name) {
let col_path = path.join(collection_name);
let col_path = get_collection_path(&path, collection_name);
if col_path.exists() {
return Ok(col_path);
}
}
return Err(Error::new(std::io::ErrorKind::Other, format!("Cant find collection: {}/{}", target_name, collection_name)));
}
Err(Error::new(std::io::ErrorKind::Other, format!("Cant find collection: {}/{}", target_name, collection_name)))
}
src/user.rs
+10 -5
View File
@@ -6,14 +6,14 @@ pub(crate) struct UserCredentials {
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct TargetUser {
pub target_name: String,
pub target: String,
pub credentials: Vec<UserCredentials>,
}
impl TargetUser {
pub fn get_target_name(&self, username: &str, password: &str) -> Option<String> {
if self.credentials.iter().find(|c| c.username.eq_ignore_ascii_case(username) && c.password.eq(password)).is_some() {
return Some(self.target_name.clone());
return Some(self.target.clone());
}
None
}
@@ -21,15 +21,20 @@ impl TargetUser {
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct User {
pub user: TargetUser,
pub user: Vec<TargetUser>,
}
impl User {
pub fn prepare(&self, _verbose: bool) {
pub fn prepare(&self) {
// TODO check if username is unique, a user can only access one target
}
pub fn get_target_name(&self, username: &str, password: &str) -> Option<String> {
self.user.get_target_name(username, password)
for target_user in &self.user {
if let Some(target_name) = target_user.get_target_name(username, password) {
return Some(target_name);
};
}
None
}
}
src/utils.rs
+32 -23
View File
@@ -1,12 +1,22 @@
use std::fs;
use std::io::{BufRead, Write};
use std::path::PathBuf;
use log::{debug, error};
use path_absolutize::*;
use reqwest::header;
use reqwest::header::{HeaderName, HeaderValue};
use crate::config::ConfigInput;
use crate::messaging::send_message;
#[macro_export]
macro_rules! exit {
($($arg:tt)*) => {{
error!($($arg)*);
std::process::exit(1);
}};
}
pub(crate) fn get_exe_path() -> PathBuf {
let default_path = std::path::PathBuf::from("./");
let current_exe = std::env::current_exe();
@@ -59,7 +69,7 @@ pub(crate) fn get_working_path(wd: &String) -> String {
Err(_) => None
}
} else {
println!("Path not found {:?}", &work_path);
error!("Path not found {:?}", &work_path);
None
}
}
@@ -72,7 +82,7 @@ pub(crate) fn get_working_path(wd: &String) -> String {
match rp.canonicalize() {
Ok(ap) => String::from(ap.to_str().unwrap_or("./")),
Err(_) => {
println!("Path not found {:?}", &rp);
error!("Path not found {:?}", &rp);
String::from("./")
}
}
@@ -84,20 +94,19 @@ pub(crate) fn open_file(file_name: &PathBuf, mandatory: bool) -> Option<fs::File
Ok(file) => Some(file),
Err(_) => {
if mandatory {
println!("cant open file: {:?}", file_name);
std::process::exit(1);
exit!("cant open file: {:?}", file_name);
}
None
}
}
}
pub(crate) fn get_input_content(working_dir: &String, url_str: &str, persist_file: Option<PathBuf>, verbose: bool) -> Option<Vec<String>> {
pub(crate) fn get_input_content(working_dir: &String, url_str: &str, persist_file: Option<PathBuf>) -> Option<Vec<String>> {
match url_str.parse::<url::Url>() {
Ok(url) => match download_content(url, persist_file, verbose) {
Ok(url) => match download_content(url, persist_file) {
Ok(content) => Some(content),
Err(e) => {
println!("cant download input url: {} => {}", url_str, e);
error!("cant download input url: {} => {}", url_str, e);
send_message(format!("Failed to download: {}", url_str).as_str());
None
}
@@ -111,7 +120,7 @@ pub(crate) fn get_input_content(working_dir: &String, url_str: &str, persist_fil
let to_file = &persist_file.unwrap();
match fs::copy(file, to_file) {
Ok(_) => {}
Err(e) => println!("cant persist to: {} => {}", to_file.to_str().unwrap_or("?"), e),
Err(e) => error!("cant persist to: {} => {}", to_file.to_str().unwrap_or("?"), e),
}
};
Some(std::io::BufReader::new(open_file(file, true).unwrap()).lines().map(|l| l.unwrap()).collect())
@@ -125,7 +134,7 @@ pub(crate) fn get_input_content(working_dir: &String, url_str: &str, persist_fil
Some(file) => Some(file),
None => {
let msg = format!("cant read input url: {:?}", &file_path.unwrap());
println!("{}", msg);
error!("{}", msg);
send_message(msg.as_str());
None
}
@@ -134,14 +143,14 @@ pub(crate) fn get_input_content(working_dir: &String, url_str: &str, persist_fil
}
}
fn download_content(url: url::Url, persist_file: Option<PathBuf>, verbose: bool) -> Result<Vec<String>, String> {
fn download_content(url: url::Url, persist_file: Option<PathBuf>) -> Result<Vec<String>, String> {
match reqwest::blocking::get(url) {
Ok(response) => {
if response.status().is_success() {
match response.text_with_charset("utf8") {
Ok(text) => {
if persist_file.is_some() {
persist_playlist(persist_file, &text, verbose);
persist_playlist(persist_file, &text);
}
let result = text.lines().map(String::from).collect();
Ok(result)
@@ -156,15 +165,15 @@ fn download_content(url: url::Url, persist_file: Option<PathBuf>, verbose: bool)
}
}
fn persist_playlist(persist_file: Option<PathBuf>, text: &String, verbose: bool) {
fn persist_playlist(persist_file: Option<PathBuf>, text: &String) {
if let Some(path_buf) = persist_file {
let filename = &path_buf.to_str().unwrap_or("?");
match fs::File::create(&path_buf) {
Ok(mut file) => match file.write_all(text.as_bytes()) {
Ok(_) => if verbose { println!("persisted: {}", filename) },
Err(e) => println!("failed to persist file {}, {}", filename, e)
Ok(_) => debug!("persisted: {}", filename),
Err(e) => error!("failed to persist file {}, {}", filename, e)
},
Err(e) => println!("failed to persist file {}, {}", filename, e)
Err(e) => error!("failed to persist file {}, {}", filename, e)
}
}
}
@@ -183,7 +192,7 @@ pub(crate) fn get_file_path(wd: &String, path: Option<PathBuf>) -> Option<PathBu
match pb.join(&p).absolutize() {
Ok(os) => Some(PathBuf::from(os)),
Err(e) => {
println!("path is not relative {:?}", e);
error!("path is not relative {:?}", e);
Some(p)
}
}
@@ -195,7 +204,7 @@ pub(crate) fn get_file_path(wd: &String, path: Option<PathBuf>) -> Option<PathBu
}
}
fn download_json_content(input: &ConfigInput, url: url::Url, persist_file: Option<PathBuf>, verbose: bool) -> Result<serde_json::Value, String> {
fn download_json_content(input: &ConfigInput, url: url::Url, persist_file: Option<PathBuf>) -> Result<serde_json::Value, String> {
let mut request = reqwest::blocking::Client::new().get(url);
if input.headers.is_empty() {
let mut headers = header::HeaderMap::new();
@@ -205,7 +214,7 @@ fn download_json_content(input: &ConfigInput, url: url::Url, persist_file: Optio
HeaderValue::from_bytes(value.as_bytes()).unwrap(),
);
}
if verbose { println!("Request with headers{:?}", &headers); }
debug!("Request with headers{:?}", &headers);
request = request.headers(headers);
}
match request.send() {
@@ -214,7 +223,7 @@ fn download_json_content(input: &ConfigInput, url: url::Url, persist_file: Optio
match response.json::<serde_json::Value>() {
Ok(content) => {
if persist_file.is_some() {
persist_playlist(persist_file, &serde_json::to_string(&content).unwrap(), verbose);
persist_playlist(persist_file, &serde_json::to_string(&content).unwrap());
}
Ok(content)
}
@@ -228,17 +237,17 @@ fn download_json_content(input: &ConfigInput, url: url::Url, persist_file: Optio
}
}
pub(crate) fn get_input_json_content(input: &ConfigInput, url_str: &String, persist_file: Option<PathBuf>, verbose: bool) -> Option<serde_json::Value> {
pub(crate) fn get_input_json_content(input: &ConfigInput, url_str: &String, persist_file: Option<PathBuf>) -> Option<serde_json::Value> {
match url_str.parse::<url::Url>() {
Ok(url) => match download_json_content(input, url, persist_file, verbose) {
Ok(url) => match download_json_content(input, url, persist_file) {
Ok(content) => Some(content),
Err(e) => {
println!("cant download input url: {} => {}", url_str, e);
error!("cant download input url: {} => {}", url_str, e);
None
}
},
Err(_) => {
println!("malformed input url: {}", url_str);
error!("malformed input url: {}", url_str);
None
}
}
src/xtream_parser.rs
+71 -34
View File
@@ -6,19 +6,10 @@ use serde::de::DeserializeOwned;
use serde_json::Value;
use crate::model_m3u::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, XtreamCluster};
use crate::model_config::{default_as_empty_str};
use log::{error};
fn default_as_empty_list() -> Vec<PlaylistItem> { vec![] }
fn null_to_default<'de, D, T>(d: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Default + Deserialize<'de>,
{
let opt = Option::deserialize(d)?;
let val = opt.unwrap_or_default();
Ok(val)
}
fn deserialize_number_from_string<'de, D, T: DeserializeOwned>(
deserializer: D,
) -> Result<Option<T>, D::Error>
@@ -53,12 +44,58 @@ fn deserialize_number_from_string<'de, D, T: DeserializeOwned>(
}
}
fn value_to_string_array(value : &Vec<Value>) -> Option<Vec<String>> {
Some(value.iter().map(|i| value_to_string(i)).filter(|i| i.is_some()).map(|i| i.unwrap()).collect())
}
fn value_to_string(v : &Value) -> Option<String> {
match v {
Value::Bool(value) => Some(value.to_string()),
Value::Number(value) => Some(value.to_string()),
Value::String(value) => Some(value.to_string()),
_ => None,
}
}
fn deserialize_as_option_string<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let value: Value = Deserialize::deserialize(deserializer)?;
match &value {
Value::String(s) => Ok(Some(s.clone())),
_ => Ok(Some(value.to_string())),
}
}
fn deserialize_as_string<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: Deserializer<'de>,
{
let value: Value = Deserialize::deserialize(deserializer)?;
match &value {
Value::String(s) => Ok(s.clone()),
_ => Ok(value.to_string()),
}
}
fn deserialize_as_string_array<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
where
D: Deserializer<'de>,
{
Value::deserialize(deserializer).map(|v| match v {
Value::Array(value) => value_to_string_array(&value),
_ => None,
})
}
#[derive(Deserialize)]
struct XtreamCategory {
#[serde(deserialize_with = "null_to_default")]
#[serde(deserialize_with = "deserialize_as_string")]
pub category_id: String,
#[serde(deserialize_with = "null_to_default")]
#[serde(deserialize_with = "deserialize_as_string")]
pub category_name: String,
//pub parent_id: i32,
#[serde(default = "default_as_empty_list")]
@@ -73,51 +110,51 @@ impl XtreamCategory {
#[derive(Deserialize)]
struct XtreamStream {
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_string")]
pub name: String,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_string")]
pub category_id: String,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub stream_id: Option<i32>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub series_id: Option<i32>,
#[serde(default = "default_as_empty_str", deserialize_with = "null_to_default")]
#[serde(default = "default_as_empty_str", deserialize_with = "deserialize_as_string")]
pub stream_icon: String,
#[serde(default = "default_as_empty_str", deserialize_with = "null_to_default")]
#[serde(default = "default_as_empty_str", deserialize_with = "deserialize_as_string")]
pub direct_source: String,
// optional attributes
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_string_array")]
backdrop_path: Option<Vec<String>>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
added: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
cast: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
container_extension: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
director: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
episode_run_time: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
genre: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
plot: Option<String>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
rating: Option<f32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
rating_5based: Option<f32>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
release_date: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
stream_type: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
title: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
year: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
youtube_trailer: Option<String>,
#[serde(default, deserialize_with = "null_to_default")]
#[serde(default, deserialize_with = "deserialize_as_option_string")]
epg_channel_id: Option<String>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
tv_archive: Option<i32>,
@@ -181,7 +218,7 @@ fn process_category(category: Option<serde_json::Value>) -> Vec<XtreamCategory>
match serde_json::from_value::<Vec<XtreamCategory>>(value) {
Ok(category_list) => category_list,
Err(err) => {
println!("Failed to process categories {}", &err);
error!("Failed to process categories {}", &err);
vec![]
}
}
@@ -197,7 +234,7 @@ fn process_streams(xtream_cluster: &XtreamCluster, streams: Option<serde_json::V
match serde_json::from_value::<Vec<XtreamStream>>(value) {
Ok(stream_list) => stream_list,
Err(err) => {
println!("Failed to process streams {:?}: {}", xtream_cluster, &err);
error!("Failed to process streams {:?}: {}", xtream_cluster, &err);
vec![]
}
}
src/xtream_player_api.rs
+3 -1
View File
@@ -28,7 +28,9 @@ pub(crate) async fn xtream_player_api(
_ => Err(std::io::Error::new(std::io::ErrorKind::Unsupported, format!("Cant find collection: {}/{}", target, &api_req.action))),
} {
Ok(file_path) => {
let file = actix_files::NamedFile::open_async(file_path).await.unwrap().set_content_type(mime::APPLICATION_JSON);
let file = actix_files::NamedFile::open_async(file_path).await.unwrap()
.set_content_type(mime::APPLICATION_JSON)
.disable_content_disposition();
file.into_response(&req)
}
Err(_) => HttpResponse::BadRequest().finish()