diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7327a69..68b13b765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ - The `kodi` format for movies can contain the `tmdb-id` (_optional_). To add the `tmdb-id` you can set now `kodi_style`, `xtream_resolve_vod`, `xtream_resolve_vod_delay`, `xtream_resolve_series` and `xtream_resolve_series_delay` to target options. - `kodi` output can now have `username` attribute to use reverse proxy mode when combined with `xtream` output. - Fixed webUI manual update for selected targets -- Added m3u logo url rewrite in `reverse proxy` mode or with `m3u_mask_redirect_url` option. +- Added m3u logo url rewrite in `reverse proxy` mode or with `m3u_mask_redirect_url` option. +- BPlusTree compression changed from zlib to zstd. - Breaking change: multi scheduler config with optional targets. ```yaml # sec min hour day of month month day of week year diff --git a/Cargo.lock b/Cargo.lock index 842680399..545837881 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1791,6 +1791,7 @@ dependencies = [ "rpassword", "rust-argon2", "rustelebot", + "ruzstd", "serde", "serde_json", "serde_yaml", @@ -2504,6 +2505,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "ruzstd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c581601827da5c717bfae77d7b187e54293d23d8fb6b700b4b5e9b5828a13cc3" +dependencies = [ + "twox-hash", +] + [[package]] name = "ryu" version = "1.0.18" @@ -2998,6 +3008,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" + [[package]] name = "typenum" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index 2294bf46a..57a16d35a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,3 +55,4 @@ tokio-stream = { version = "0.1", features = ["sync"] } tokio = "1.41" paste = "1.0" tempfile = "3.14" +ruzstd = "0" diff --git a/src/model/xtream.rs b/src/model/xtream.rs index 6e608b71f..1a4bb5c41 100644 --- a/src/model/xtream.rs +++ b/src/model/xtream.rs @@ -423,11 +423,11 @@ impl XtreamSeriesEpisode { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct XtreamSeriesInfo { #[serde(default)] - pub seasons: Vec, + pub seasons: Option>, #[serde(default)] pub info: Option, #[serde(default)] - pub episodes: HashMap>, + pub episodes: Option>>, } diff --git a/src/processing/xtream_parser.rs b/src/processing/xtream_parser.rs index ddea7d66d..7a146120a 100644 --- a/src/processing/xtream_parser.rs +++ b/src/processing/xtream_parser.rs @@ -10,7 +10,6 @@ use crate::model::config::ConfigInput; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream}; use crate::repository::storage::hash_string; -use crate::utils::request_utils::mask_sensitive_info; fn map_to_xtream_category(categories: &Value) -> Result, M3uFilterError> { match serde_json::from_value::>(categories.to_owned()) { @@ -47,30 +46,33 @@ pub fn parse_xtream_series_info(info: &Value, group_title: &str, series_name: &s match serde_json::from_value::(info.to_owned()) { Ok(series_info) => { - let result: Vec<(XtreamSeriesInfoEpisode, PlaylistItem)> = series_info.episodes.values().flatten().map(|episode| { - let episode_url = create_xtream_series_info_url(url, username, password, episode); - (episode.clone(), - PlaylistItem { - header: RefCell::new(PlaylistItemHeader { - id: Rc::new(episode.id.to_string()), - uuid: Rc::new(hash_string(&episode_url)), - name: Rc::new(series_name.to_string()), - logo: Rc::new(episode.info.as_ref().map_or_else(String::new, |info| info.movie_image.to_string())), - group: Rc::new(group_title.to_string()), - title: Rc::new(episode.title.clone()), - url: episode_url, - item_type: PlaylistItemType::Series, - xtream_cluster: XtreamCluster::Series, - additional_properties: episode.get_additional_properties(&series_info), - category_id: 0, - input_id: input.id, - ..Default::default() - }) - })}).collect(); - if result.is_empty() { Ok(None) } else { Ok(Some(result)) } + if let Some(episodes) = &series_info.episodes { + let result: Vec<(XtreamSeriesInfoEpisode, PlaylistItem)> = episodes.values().flatten().map(|episode| { + let episode_url = create_xtream_series_info_url(url, username, password, episode); + (episode.clone(), + PlaylistItem { + header: RefCell::new(PlaylistItemHeader { + id: Rc::new(episode.id.to_string()), + uuid: Rc::new(hash_string(&episode_url)), + name: Rc::new(series_name.to_string()), + logo: Rc::new(episode.info.as_ref().map_or_else(String::new, |info| info.movie_image.to_string())), + group: Rc::new(group_title.to_string()), + title: Rc::new(episode.title.clone()), + url: episode_url, + item_type: PlaylistItemType::Series, + xtream_cluster: XtreamCluster::Series, + additional_properties: episode.get_additional_properties(&series_info), + category_id: 0, + input_id: input.id, + ..Default::default() + }) + })}).collect(); + return if result.is_empty() { Ok(None) } else { Ok(Some(result)) }; + } + Ok(None) } Err(err) => { - create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "Failed to process series info {} {}", &err, mask_sensitive_info(url)) + create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "Failed to process series info for {series_name} {err}") } } } diff --git a/src/repository/bplustree.rs b/src/repository/bplustree.rs index b7e5eec9d..7240fc4d5 100644 --- a/src/repository/bplustree.rs +++ b/src/repository/bplustree.rs @@ -5,16 +5,15 @@ use std::marker::PhantomData; use std::mem::size_of; use std::path::Path; -use flate2::Compression; use log::error; +use ruzstd::decoding::StreamingDecoder; +use ruzstd::encoding::{compress_to_vec, CompressionLevel}; use serde::{Deserialize, Serialize}; use tempfile::NamedTempFile; use crate::utils::file_utils::{open_read_write_file, rename_or_copy}; -// TODO replace ZLib with faster compression like lz4 or zstd - -const BINCODE_OVERHEAD: usize = 8; const BLOCK_SIZE: usize = 4096; +const BINCODE_OVERHEAD: usize = 8; const LEN_SIZE: usize = 4; const FLAG_SIZE: usize = 1; @@ -236,9 +235,10 @@ where buffer_slice[write_pos..=write_pos].copy_from_slice(&compression_byte); write_pos += FLAG_SIZE; let content_bytes = if use_compression { - let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::fast()); - encoder.write_all(&values_encoded)?; - encoder.finish()? + { + // use ruzstd::io::Read; + compress_to_vec(&*values_encoded, CompressionLevel::Fastest) + } } else { values_encoded }; @@ -337,14 +337,12 @@ where }; let values_bytes = if use_compression { - let mut decoder = flate2::write::ZlibDecoder::new(Vec::new()); - decoder.write_all(&content_bytes)?; - &decoder.finish()? + decode_content(&content_bytes).unwrap_or(content_bytes) } else { - &content_bytes + content_bytes }; - let values: Vec = bincode_deserialize(values_bytes)?; + let values: Vec = bincode_deserialize(&values_bytes)?; read_pos += values_length; values } else { @@ -377,6 +375,24 @@ where } } +fn decode_content(content_bytes: &Vec) -> Option> { + if let Ok(mut decoder) = StreamingDecoder::new(&**content_bytes) { + let mut result = Vec::new(); + if decoder.read_to_end(&mut result).is_ok() { + return Some(result) + } + } + + // TODO remove at next deployment, this is only fallback for older compressed files + let mut decoder = flate2::write::ZlibDecoder::new(Vec::new()); + if let Ok(()) = decoder.write_all(content_bytes) { + if let Ok(decoded) = decoder.finish() { + return Some(decoded); + } + } + None +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct BPlusTree { root: BPlusTreeNode,