From 5d5b98f60f4fb602addc742dac1c5bcd1624abfd Mon Sep 17 00:00:00 2001 From: euzu Date: Tue, 2 Apr 2024 16:17:49 +0200 Subject: [PATCH] refactored reverse proxy implementation for m3u playlist for better performance --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + src/processing/m3u_parser.rs | 10 +++++----- src/repository/m3u_repository.rs | 7 ++++--- src/utils/download.rs | 2 +- src/utils/file_reader.rs | 24 ++++++++++++++++++++++++ src/utils/mod.rs | 3 ++- 7 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 src/utils/file_reader.rs diff --git a/Cargo.lock b/Cargo.lock index c92ca984f..f62abca93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1313,6 +1313,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linereader" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d921fea6860357575519aca014c6e22470585accdd543b370c404a8a72d0dd1d" +dependencies = [ + "memchr", +] + [[package]] name = "linux-raw-sys" version = "0.4.13" @@ -1378,6 +1387,7 @@ dependencies = [ "enum-iterator", "env_logger", "futures", + "linereader", "log", "lzma-rs", "mime", diff --git a/Cargo.toml b/Cargo.toml index aa4bbf35a..7daff82d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,3 +43,4 @@ rustelebot = "0.3" bincode = "1.3" uuid = { version = "1.7", features = ["v4", "fast-rng", "macro-diagnostics"] } lzma-rs = "0.3" +linereader = "0" \ No newline at end of file diff --git a/src/processing/m3u_parser.rs b/src/processing/m3u_parser.rs index 5718b1dfa..14934b998 100644 --- a/src/processing/m3u_parser.rs +++ b/src/processing/m3u_parser.rs @@ -160,14 +160,14 @@ fn extract_id_from_url(url: &str) -> Option { None } -pub(crate) fn consume_m3u(cfg: &Config, lines: &Vec, mut visit: F) { +pub(crate) fn consume_m3u(cfg: &Config, lines: impl Iterator, mut visit: F) { let mut header: Option = None; let mut group: Option = None; let video_suffixes = cfg.video.as_ref().unwrap().extensions.iter().map(|ext| ext.as_str()).collect(); for line in lines { if line.starts_with("#EXTINF") { - header = Some(String::from(line)); + header = Some(line); continue; } if line.starts_with("#EXTGRP") { @@ -178,7 +178,7 @@ pub(crate) fn consume_m3u(cfg: &Config, lines: &Vec(cfg: &Config, lines: &Vec) -> Vec { +pub(crate) fn parse_m3u(cfg: &Config, lines: &[String]) -> Vec { let mut groups: std::collections::HashMap, Vec> = std::collections::HashMap::new(); let mut sort_order: Vec> = vec![]; let mut playlist = Vec::new(); - consume_m3u(cfg, lines, |item| playlist.push(item)); + consume_m3u(cfg, lines.iter().cloned(), |item| playlist.push(item)); playlist.drain(..).for_each(|item| { let key = Rc::clone(&item.header.borrow().group); // let key2 = String::from(&item.header.group); diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 35fea23a3..852bdcb1d 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::{Error, BufRead, BufReader, ErrorKind, Write, SeekFrom, Seek, Read}; +use std::io::{Error, ErrorKind, Write, SeekFrom, Seek, Read}; use std::path::Path; use std::rc::Rc; @@ -13,6 +13,7 @@ use crate::model::api_proxy::UserCredentials; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{PlaylistGroup, PlaylistItemType}; use crate::processing::m3u_parser::consume_m3u; +use crate::utils::file_reader::FileReader; use crate::utils::file_utils; macro_rules! cant_write_result { @@ -251,8 +252,8 @@ pub(crate) fn rewrite_m3u_playlist(cfg: &Config, target: &ConfigTarget, user: &U result.push("#EXTM3U".to_string()); match File::open(&m3u_path) { Ok(m3u_file) => { - let lines: Vec = BufReader::new(m3u_file).lines().map_while(Result::ok).collect(); - consume_m3u(cfg, &lines, |item| { + let reader = FileReader::new(m3u_file); + consume_m3u(cfg, reader, |item| { let stream_id = item.header.borrow().stream_id; item.header.borrow_mut().url = Rc::new(format!("{}/{}", url, stream_id)); result.push(item.to_m3u(target)) diff --git a/src/utils/download.rs b/src/utils/download.rs index 1731eee83..67138839f 100644 --- a/src/utils/download.rs +++ b/src/utils/download.rs @@ -33,7 +33,7 @@ pub(crate) async fn get_m3u_playlist(cfg: &Config, input: &ConfigInput, working_ let persist_file_path = prepare_file_path(input, working_dir, ""); match request_utils::get_input_text_content(input, working_dir, &url, persist_file_path).await { Ok(text) => { - let lines = text.lines().map(String::from).collect(); + let lines = text.lines().map(String::from).collect::>(); (m3u_parser::parse_m3u(cfg, &lines), vec![]) } Err(err) => (vec![], vec![err]) diff --git a/src/utils/file_reader.rs b/src/utils/file_reader.rs new file mode 100644 index 000000000..27e2104b1 --- /dev/null +++ b/src/utils/file_reader.rs @@ -0,0 +1,24 @@ +use std::fs::File; +use linereader::LineReader; + +pub(crate) struct FileReader { + reader: LineReader, +} + +impl FileReader { + pub(crate) fn new(file: File) -> FileReader { + FileReader { + reader: LineReader::new(file), + } + } +} + +impl Iterator for FileReader { + type Item = String; + fn next(&mut self) -> Option { + if let Some(Ok(buf)) = self.reader.next_line() { + return Some(String::from_utf8_lossy(buf).trim_end_matches(char::is_control).to_string()); + } + None + } +} \ No newline at end of file diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 53fe4fb75..9fb33a6ed 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -4,4 +4,5 @@ pub (crate) mod download; pub (crate) mod string_utils; pub (crate) mod json_utils; pub (crate) mod config_reader; -pub (crate) mod multi_file_reader; \ No newline at end of file +pub (crate) mod multi_file_reader; +pub (crate) mod file_reader; \ No newline at end of file