From c8586b57b14ffdbc775ebfb8e6bb3f2685c0fdc7 Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 25 Oct 2024 00:39:22 +0200 Subject: [PATCH 01/11] added virtual id handling --- Cargo.lock | 94 ++++++----- Cargo.toml | 8 +- src/api/m3u_api.rs | 23 ++- src/api/xmltv_api.rs | 6 +- src/api/xtream_api.rs | 21 +-- src/main.rs | 9 +- src/model/playlist.rs | 69 ++++---- src/model/xtream.rs | 43 +++-- src/processing/m3u_parser.rs | 51 +++--- src/processing/playlist_processor.rs | 69 ++++---- src/processing/xtream_parser.rs | 105 ++++++------ src/repository/bplustree.rs | 181 --------------------- src/repository/epg_repository.rs | 4 +- src/repository/index_record.rs | 17 -- src/repository/indexed_document_writer.rs | 4 +- src/repository/m3u_repository.rs | 122 +++++++------- src/repository/mod.rs | 8 +- src/repository/playlist_repository.rs | 54 ++++-- src/repository/storage.rs | 30 ++++ src/repository/target_id_mapping_record.rs | 141 ++++++++++++++++ src/repository/xtream_repository.rs | 16 +- src/test.rs | 25 +-- src/utils/default_utils.rs | 3 - src/utils/download.rs | 47 +++--- src/utils/request_utils.rs | 24 +-- 25 files changed, 587 insertions(+), 587 deletions(-) delete mode 100644 src/repository/bplustree.rs create mode 100644 src/repository/storage.rs create mode 100644 src/repository/target_id_mapping_record.rs diff --git a/Cargo.lock b/Cargo.lock index 5122f908d..f2f4e0b83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,9 +506,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bytestring" @@ -527,9 +527,9 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" [[package]] name = "cc" -version = "1.1.30" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "jobserver", "libc", @@ -602,6 +602,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "compressed_string" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3191fb9cd59eb0195c223d4f1d8894d2078957ee44f4940f059d11060a6d0e7" +dependencies = [ + "flate2", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -760,9 +769,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -1259,9 +1268,9 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e658178c10c747241199382079c0f195ce229866fbf4aa0d46fa6107fe33d2ec" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" [[package]] name = "indexmap" @@ -1370,9 +1379,9 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libnghttp2-sys" @@ -1445,11 +1454,11 @@ dependencies = [ "actix-server", "actix-web", "actix-web-httpauth", - "base64 0.22.1", "bincode", "blake3", "chrono", "clap", + "compressed_string", "cron", "enum-iterator", "env_logger", @@ -1607,9 +1616,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" -version = "0.10.67" +version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8cefcf97f41316955f9294cd61f639bdcfa9f2f230faac6cb896aa8ab64704" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -1639,9 +1648,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.3.2+3.3.2" +version = "300.4.0+3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b" +checksum = "a709e02f2b4aca747929cca5ed248880847c650233cf8b8cdc48f40aaf4898a6" dependencies = [ "cc", ] @@ -1785,18 +1794,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", @@ -1805,9 +1814,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -1854,9 +1863,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -2149,9 +2158,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "once_cell", "ring", @@ -2239,18 +2248,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", @@ -2259,9 +2268,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2405,9 +2414,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.79" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -2459,18 +2468,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", @@ -2525,9 +2534,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -2642,12 +2651,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" diff --git a/Cargo.toml b/Cargo.toml index ba849560a..2d1b392fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ strip = true [dependencies] serde = { version = "1.0", features = ["derive", "rc"] } -serde_yaml = "0.9.33" +serde_yaml = "0.9.34" serde_json = "1" quick-xml = { version = "0.36", features = ["serialize"] } -regex = "1.10" +regex = "1.11" clap = { version = "4", features = ["derive"] } url = "2.5" reqwest = { version = "0", features = ["blocking", "json", "stream", "rustls-tls"] } @@ -49,5 +49,5 @@ rand = "0.8" rpassword = "7.3" flate2 = "1" time = "0.3" -blake3 = "1.5.4" -base64 = "0.22.1" +blake3 = "1.5" +compressed_string = "1.0.0" diff --git a/src/api/m3u_api.rs b/src/api/m3u_api.rs index 94d72fd98..1c2bf70c9 100644 --- a/src/api/m3u_api.rs +++ b/src/api/m3u_api.rs @@ -5,6 +5,7 @@ use crate::api::api_utils::{get_user_target, get_user_target_by_credentials, str use crate::api::api_model::{AppState, UserApiRequest}; use crate::model::config::TargetType; use crate::repository::m3u_repository::{m3u_get_file_paths, m3u_get_item_for_stream_id, m3u_load_rewrite_playlist}; +use crate::repository::storage::get_target_storage_path; async fn m3u_api( api_req: web::Query, @@ -38,15 +39,23 @@ async fn m3u_api_stream( if let Ok(m3u_stream_id) = stream_id.parse::() { if let Some((_user, target)) = get_user_target_by_credentials(&username, &password, &api_req, &app_state) { if target.has_output(&TargetType::M3u) { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&app_state.config, target) { - match m3u_get_item_for_stream_id(m3u_stream_id, &m3u_path, &idx_path) { - Ok(m3u_item) => { - return stream_response(m3u_item.url.as_str(), &req, None).await; - } - Err(err) => { - error!("Failed to get m3u url: {}", err); + + match get_target_storage_path(&app_state.config, target.name.as_str()) { + Some(target_path) => { + if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) { + match m3u_get_item_for_stream_id(m3u_stream_id, &m3u_path, &idx_path) { + Ok(m3u_item) => { + return stream_response(m3u_item.url.as_str(), &req, None).await; + } + Err(err) => { + error!("Failed to get m3u url: {}", err); + } + } } } + None => { + error!("Failed to get target path for {}", target.name); + } } } } diff --git a/src/api/xmltv_api.rs b/src/api/xmltv_api.rs index 4a54e9427..ad310cff9 100644 --- a/src/api/xmltv_api.rs +++ b/src/api/xmltv_api.rs @@ -17,6 +17,7 @@ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigInput, ConfigTarget}; use crate::model::config::TargetType; use crate::repository::m3u_repository::m3u_get_epg_file_path; +use crate::repository::storage::get_target_storage_path; use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path}; use crate::utils::{file_utils, request_utils}; @@ -51,10 +52,13 @@ fn get_epg_path_for_target_of_type(target_name: &str, file_path: Option } fn get_epg_path_for_target(config: &Config, target: &ConfigTarget) -> Option { + // TODO if we share the same virtual_id for epg, can we store an epg file for the target ? for output in &target.output { match output.target { TargetType::M3u => { - return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path(config, target)); + if let Some(target_path) = get_target_storage_path(config, &target.name) { + return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path(&target_path)); + } } TargetType::Xtream => { if let Some(storage_path) = xtream_get_storage_path(config, &target.name) { diff --git a/src/api/xtream_api.rs b/src/api/xtream_api.rs index 972701d69..12fb38130 100644 --- a/src/api/xtream_api.rs +++ b/src/api/xtream_api.rs @@ -263,7 +263,7 @@ async fn xtream_player_api_streaming_timeshift( fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result { if let Ok(mut doc) = serde_json::from_str::>(content) { if let Some(Value::Object(movie_data)) = doc.get_mut("movie_data") { - let stream_id = pli.stream_id; + let stream_id = pli.virtual_id; let category_id = pli.category_id; movie_data.insert("stream_id".to_string(), Value::Number(serde_json::value::Number::from(stream_id))); movie_data.insert("category_id".to_string(), Value::Number(serde_json::value::Number::from(category_id))); @@ -278,7 +278,7 @@ fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: } } } - Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.stream_id))) + Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.virtual_id))) } fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result { @@ -309,13 +309,13 @@ fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPl } } if let Ok(result) = serde_json::to_string(&doc) { - let _ = xtream_repository::xtream_write_series_info(config, target.name.replace(' ', "_").as_str(), pli.stream_id, &new_id_to_provider_id_mapping, &result); + let _ = xtream_repository::xtream_write_series_info(config, target.name.replace(' ', "_").as_str(), pli.virtual_id, &new_id_to_provider_id_mapping, &result); return Ok(result); } } } } - Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli.stream_id))) + Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli.virtual_id))) } async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> Result { @@ -325,7 +325,7 @@ async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &ConfigTarget, pli: &XtreamPlaylistItem, info_url: &str, cluster: XtreamCluster) -> Result { if cluster == XtreamCluster::Series { - if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.replace(' ', "_").as_str(), pli.stream_id) { + if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.replace(' ', "_").as_str(), pli.virtual_id) { return Ok(content); } } @@ -343,7 +343,7 @@ async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &C } Err(Error::new(std::io::ErrorKind::Other, format!("Cant find stream with id: {}/{}/{}", - target.name.replace(' ', "_").as_str(), &cluster, pli.stream_id))) + target.name.replace(' ', "_").as_str(), &cluster, pli.virtual_id))) } async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserCredentials, @@ -549,13 +549,8 @@ async fn xtream_player_api( } } _ => { - if api_req.action.is_empty() { - debug!("Paremeter action is empty!"); - HttpResponse::Unauthorized().finish() - } else { - debug!("cant find user!"); - HttpResponse::BadRequest().finish() - } + debug!("{}", if api_req.action.is_empty() { "Paremeter action is empty!" } else { "cant find user!" }); + HttpResponse::BadRequest().finish() } } } diff --git a/src/main.rs b/src/main.rs index bc28432bd..ca68cf275 100644 --- a/src/main.rs +++ b/src/main.rs @@ -147,11 +147,10 @@ fn init_logger(log_level: &str) { let mut log_builder = Builder::from_default_env(); if log_level.contains('=') { - let pairs: Vec<&str> = log_level.split(',').collect(); - for pair in pairs { - let kv: Vec<&str> = pair.split('=').collect(); - if kv.len() == 2 { - log_builder.filter_module(kv[0].trim(), get_log_level(kv[1].trim())); + for pair in log_level.split(',').filter(|s| s.contains('=')) { + let mut kv_iter = pair.split('=').map(str::trim); + if let (Some(module), Some(level)) = (kv_iter.next(), kv_iter.next()) { + log_builder.filter_module(module, get_log_level(level)); } } } else { diff --git a/src/model/playlist.rs b/src/model/playlist.rs index 7ab5c4ebb..ec883a60f 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -2,9 +2,6 @@ use std::cell::RefCell; use std::cmp::PartialEq; use std::fmt::{Display, Formatter}; use std::rc::Rc; -use base64::Engine; -use blake3::Hasher; -use base64::engine::general_purpose; use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; @@ -12,8 +9,8 @@ use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{ConfigInput, ConfigTarget}; use crate::model::xmltv::TVGuide; use crate::model::xtream::{xtream_playlistitem_to_document, XtreamMappingOptions}; -use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32, default_playlist_item_type, default_stream_cluster}; -use crate::utils::request_utils::get_base_url; +use crate::repository::storage::hash_string; +use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32}; // https://de.wikipedia.org/wiki/M3U // https://siptv.eu/howto/playlist.html @@ -44,6 +41,12 @@ pub(crate) enum XtreamCluster { Series = 3, } +impl Default for XtreamCluster { + fn default() -> Self { + XtreamCluster::Live + } +} + impl Display for XtreamCluster { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", match self { @@ -62,6 +65,22 @@ pub(crate) enum PlaylistItemType { SeriesInfo = 4, } +impl Default for PlaylistItemType { + fn default() -> Self { + PlaylistItemType::Live + } +} + +impl From for PlaylistItemType { + fn from(xtream_cluster: XtreamCluster) -> Self { + match xtream_cluster { + XtreamCluster::Live => PlaylistItemType::Live, + XtreamCluster::Video => PlaylistItemType::Movie, + XtreamCluster::Series => PlaylistItemType::SeriesInfo, + } + } +} + impl Display for PlaylistItemType { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", match self { @@ -78,11 +97,11 @@ pub(crate) trait FieldAccessor { fn set_field(&mut self, field: &str, value: &str) -> bool; } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct PlaylistItemHeader { - pub uuid: Rc, // calculated - pub stream_id: Rc, // virtual id + pub uuid: Rc<[u8;32]>, // calculated pub id: Rc, // provider id + pub virtual_id: u32, // virtual id pub name: Rc, pub chno: Rc, pub logo: Rc, @@ -95,10 +114,9 @@ pub(crate) struct PlaylistItemHeader { pub rec: Rc, pub url: Rc, pub epg_channel_id: Option>, - #[serde(default = "default_stream_cluster")] pub xtream_cluster: XtreamCluster, pub additional_properties: Option, - #[serde(default = "default_playlist_item_type", skip_serializing, skip_deserializing)] + #[serde(skip_serializing, skip_deserializing)] pub item_type: PlaylistItemType, #[serde(default = "default_as_false", skip_serializing, skip_deserializing)] pub series_fetched: bool, // only used for series_info @@ -111,20 +129,10 @@ pub(crate) struct PlaylistItemHeader { impl PlaylistItemHeader { pub(crate) fn gen_uuid(&mut self) { - //let base_url = get_base_url(&self.url).unwrap_or_else(|| self.url.to_string()); - // Create a Blake3 hasher - let mut hasher = Hasher::new(); - // the url should be different for each entry. - hasher.update(self.url.as_bytes()); - // Finalize and get the hash result - let hash_bytes = hasher.finalize(); - // Encode the reduced 128-bit hash to Base62 - let mut encoded_key = String::new(); - general_purpose::STANDARD.encode_string(&hash_bytes.as_bytes(), &mut encoded_key); - self.uuid = Rc::new(encoded_key); + self.uuid = Rc::new(hash_string(&self.url)) } - pub(crate) fn get_uuid(&self) -> &str { - self.uuid.as_ref() + pub(crate) fn get_uuid(&self) -> &Rc<[u8; 32]> { + &self.uuid } } @@ -172,11 +180,11 @@ macro_rules! generate_field_accessor_impl_for_playlist_item_header { } } -generate_field_accessor_impl_for_playlist_item_header!(id, stream_id, name, chno, logo, logo_small, group, title, parent_code, audio_track, time_shift, rec, url;); +generate_field_accessor_impl_for_playlist_item_header!(id, /*virtual_id,*/ name, chno, logo, logo_small, group, title, parent_code, audio_track, time_shift, rec, url;); #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct M3uPlaylistItem { - pub stream_id: Rc, + pub virtual_id: u32, pub provider_id: Rc, pub name: Rc, pub chno: Rc, @@ -218,7 +226,7 @@ impl M3uPlaylistItem { #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct XtreamPlaylistItem { - pub stream_id: u32, + pub virtual_id: u32, pub provider_id: u32, pub name: Rc, pub logo: Rc, @@ -252,7 +260,7 @@ impl PlaylistItem { pub fn to_m3u(&self) -> M3uPlaylistItem { let header = self.header.borrow(); M3uPlaylistItem { - stream_id: Rc::clone(&header.stream_id), + virtual_id: header.virtual_id, provider_id: Rc::clone(&header.id), name: Rc::clone(&header.name), chno: Rc::clone(&header.chno), @@ -275,7 +283,7 @@ impl PlaylistItem { match header.id.parse::() { Ok(provider_id) => { Ok(XtreamPlaylistItem { - stream_id: header.stream_id.parse::().unwrap_or(0), + virtual_id: header.virtual_id, provider_id, name: Rc::clone(&header.name), logo: Rc::clone(&header.logo), @@ -312,7 +320,7 @@ pub(crate) struct PlaylistGroup { pub id: u32, pub title: Rc, pub channels: Vec, - #[serde(default = "default_stream_cluster", skip_serializing, skip_deserializing)] + #[serde(skip_serializing, skip_deserializing)] pub xtream_cluster: XtreamCluster, } @@ -321,5 +329,4 @@ impl PlaylistGroup { pub(crate) fn on_load(&mut self) { self.channels.iter().for_each(|pl| pl.header.borrow_mut().gen_uuid()); } -} - +} \ No newline at end of file diff --git a/src/model/xtream.rs b/src/model/xtream.rs index ba6522615..c1c199f3b 100644 --- a/src/model/xtream.rs +++ b/src/model/xtream.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use crate::model::config::ConfigTargetOptions; use crate::model::playlist::{PlaylistItem, XtreamCluster, XtreamPlaylistItem}; -use crate::utils::default_utils::{default_as_empty_rc_str, default_as_empty_list}; +use crate::utils::default_utils::{default_as_empty_list, default_as_empty_rc_str}; const LIVE_STREAM_FIELDS: &[&str] = &[]; @@ -28,8 +28,8 @@ const SERIES_STREAM_FIELDS: &[&str] = &[ fn deserialize_number_from_string<'de, D, T: DeserializeOwned>( deserializer: D, ) -> Result, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { // we define a local enum type inside of the function // because it is untagged, serde will deserialize as the first variant @@ -73,8 +73,8 @@ fn value_to_string(v: &Value) -> Option { } fn deserialize_as_option_rc_string<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { let value: Value = Deserialize::deserialize(deserializer)?; @@ -86,8 +86,8 @@ fn deserialize_as_option_rc_string<'de, D>(deserializer: D) -> Result(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { let value: Value = Deserialize::deserialize(deserializer)?; @@ -98,8 +98,8 @@ fn deserialize_as_rc_string<'de, D>(deserializer: D) -> Result, D::Er } fn deserialize_as_string_array<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { Value::deserialize(deserializer).map(|v| match v { Value::Array(value) => Some(value_to_string_array(&value)), @@ -131,9 +131,9 @@ pub(crate) struct XtreamStream { #[serde(default, deserialize_with = "deserialize_as_rc_string")] pub category_id: Rc, #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub stream_id: Option, + pub stream_id: Option, #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub series_id: Option, + pub series_id: Option, #[serde(default = "default_as_empty_rc_str", deserialize_with = "deserialize_as_rc_string")] pub stream_icon: Rc, #[serde(default = "default_as_empty_rc_str", deserialize_with = "deserialize_as_rc_string")] @@ -217,10 +217,9 @@ macro_rules! add_i64_property_if_exists { } } - impl XtreamStream { - pub(crate) fn get_stream_id(&self) -> String { - self.stream_id.map_or_else(|| self.series_id.map_or_else(String::new, |seid| format!("{seid}")), |sid| format!("{sid}")) + pub(crate) fn get_stream_id(&self) -> u32 { + self.stream_id.unwrap_or(self.series_id.unwrap_or(0)) } pub(crate) fn get_additional_properties(&self) -> Option { @@ -315,6 +314,18 @@ pub(crate) struct XtreamSeriesInfoEpisode { pub direct_source: String, } +// impl XtreamSeriesInfoEpisode { +// pub(crate) fn get_id(&self) -> u32 { +// match self.id.parse::() { +// Ok(id) => id, +// Err(_) => { +// error!("Failed to convert id to number {}", self.id); +// 0 +// } +// } +// } +// } + #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct XtreamSeriesInfo { pub seasons: Vec, @@ -358,7 +369,7 @@ impl XtreamMappingOptions { pub fn from_target_options(options: Option<&ConfigTargetOptions>) -> Self { let (skip_live_direct_source, skip_video_direct_source, skip_series_direct_source) = options .map_or((false, false, false), |o| (o.xtream_skip_live_direct_source, - o.xtream_skip_video_direct_source, o.xtream_skip_series_direct_source)); + o.xtream_skip_video_direct_source, o.xtream_skip_series_direct_source)); Self { skip_live_direct_source, skip_video_direct_source, @@ -414,7 +425,7 @@ fn append_prepared_series_properties(add_props: Option<&Map>, doc } pub(crate) fn xtream_playlistitem_to_document(pli: &XtreamPlaylistItem, options: &XtreamMappingOptions) -> serde_json::Value { - let stream_id_value = Value::Number(serde_json::Number::from(pli.stream_id)); + let stream_id_value = Value::Number(serde_json::Number::from(pli.virtual_id)); let mut document = serde_json::Map::from_iter([ ("category_id".to_string(), Value::String(format!("{}", &pli.category_id))), ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(pli.category_id))]))), diff --git a/src/processing/m3u_parser.rs b/src/processing/m3u_parser.rs index 6c4b05231..ba1cb96e0 100644 --- a/src/processing/m3u_parser.rs +++ b/src/processing/m3u_parser.rs @@ -4,7 +4,6 @@ use std::rc::Rc; use crate::model::config::{Config, ConfigInput}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use crate::utils::default_utils::{default_as_empty_rc_str, default_playlist_item_type, default_stream_cluster}; use crate::utils::string_utils; #[inline] @@ -76,27 +75,10 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { fn create_empty_playlistitem_header(input_id: u16, url: &str) -> PlaylistItemHeader { PlaylistItemHeader { - uuid: default_as_empty_rc_str(), - id: default_as_empty_rc_str(), - stream_id: default_as_empty_rc_str(), - name: default_as_empty_rc_str(), - chno: default_as_empty_rc_str(), - logo: default_as_empty_rc_str(), - logo_small: default_as_empty_rc_str(), - group: default_as_empty_rc_str(), - title: default_as_empty_rc_str(), - parent_code: default_as_empty_rc_str(), - audio_track: default_as_empty_rc_str(), - time_shift: default_as_empty_rc_str(), - rec: default_as_empty_rc_str(), url: Rc::new(url.to_owned()), - epg_channel_id: None, - item_type: default_playlist_item_type(), - xtream_cluster: default_stream_cluster(), - additional_properties: None, - series_fetched: false, category_id: 0, input_id, + ..Default::default() } } @@ -147,7 +129,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str plih.id = Rc::new(chanid); } } - plih.stream_id = Rc::clone(&plih.id); + // plih.virtual_id = plih.id; plih.epg_channel_id = Some(Rc::clone(&plih.id)); } @@ -174,7 +156,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str pub(crate) fn extract_id_from_url(url: &str) -> Option { if let Some(filename) = url.split('/').last() { - return if let Some(index) = filename.rfind('.') { + return if let Some(index) = filename.rfind('.') { Some(filename[..index].to_string()) } else { Some(filename.to_string()) @@ -183,14 +165,17 @@ pub(crate) fn extract_id_from_url(url: &str) -> Option { None } -pub(crate) fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: impl Iterator, mut visit: F) { +pub(crate) fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F) +where + I: Iterator, +{ let mut header: Option = None; let mut group: Option = None; let video_suffixes = cfg.video.as_ref().unwrap().extensions.iter().map(String::as_str).collect::>(); for line in lines { if line.starts_with("#EXTINF") { - header = Some(line); + header = Some(String::from(line)); continue; } if line.starts_with("#EXTGRP") { @@ -219,29 +204,33 @@ pub(crate) fn consume_m3u(cfg: &Config, input: &ConfigIn } } -pub(crate) fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: &[String]) -> Vec { +pub(crate) fn parse_m3u<'a, I>(cfg: &Config, input: &ConfigInput, lines: I) -> Vec +where + I: Iterator, +{ let mut sort_order: Vec> = vec![]; - let mut idx: usize = 0; + let mut sort_order_idx: usize = 0; let mut group_map: std::collections::HashMap, usize> = std::collections::HashMap::new(); - consume_m3u(cfg, input, lines.iter().cloned(), |item| { + consume_m3u(cfg, input, lines, |item| { + // keep the original sort order for groups and group the playlist items let key = Rc::clone(&item.header.borrow().group); match group_map.entry(key) { std::collections::hash_map::Entry::Vacant(v) => { - v.insert(idx); - idx += 1; + v.insert(sort_order_idx); sort_order.push(vec![item]); + sort_order_idx += 1; } std::collections::hash_map::Entry::Occupied(o) => { sort_order.get_mut(*o.get()).unwrap().push(item); } } }); - let mut grp_id = 0; let result: Vec = sort_order.drain(..).map(|channels| { + // create a group based on the first playlist item let channel = channels.first(); - let cluster = channel.map(|pli| pli.header.borrow().xtream_cluster).unwrap(); - let group_title = channel.map(|pli| Rc::clone(&pli.header.borrow().group)).unwrap(); + let (cluster, group_title) = channel.map(|pli| + (pli.header.borrow().xtream_cluster, Rc::clone(&pli.header.borrow().group))).unwrap(); grp_id += 1; PlaylistGroup { id: grp_id, xtream_cluster: cluster, title: Rc::clone(&group_title), channels } }).collect(); diff --git a/src/processing/playlist_processor.rs b/src/processing/playlist_processor.rs index 57cffdb51..c6d9b4d2b 100644 --- a/src/processing/playlist_processor.rs +++ b/src/processing/playlist_processor.rs @@ -25,8 +25,8 @@ use crate::processing::playlist_watch::process_group_watch; use crate::processing::xmltv_parser::flatten_tvguide; use crate::processing::xtream_processor::playlist_resolve_series; use crate::repository::playlist_repository::persist_playlist; -use crate::utils::download; use crate::utils::default_utils::default_as_default; +use crate::utils::download; fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool { let provider = ValueProvider { pli: RefCell::new(pli) }; @@ -96,7 +96,7 @@ fn playlistitem_comparator(a: &PlaylistItem, b: &PlaylistItem, channel_sort: &Co } } } - }, + } None => { let ordering = value_a.partial_cmp(&value_b).unwrap(); match channel_sort.order { @@ -255,7 +255,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) { for channel in &plg.channels { let provider = ValueProvider { pli: RefCell::new(channel) }; if counter.filter.filter(&provider, &mut mock_processor) { - let new_value = if counter.modifier == CounterModifier::Assign { + let new_value = if counter.modifier == CounterModifier::Assign { cntval.to_string() } else { let value = match channel.header.borrow_mut().get_field(&counter.field) { @@ -264,7 +264,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) { }; if counter.modifier == CounterModifier::Suffix { format!("{}{}{}", value, counter.concat, cntval.to_string()) - } else { + } else { format!("{}{}{}", cntval.to_string(), counter.concat, value) } }; @@ -295,10 +295,11 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc) -> (Vec, Vec) { let source = cfg.sources.get(source_idx).unwrap(); - let mut source_playlists = Vec::new(); - let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count(); let mut errors = vec![]; let mut stats = HashMap::::new(); + let mut source_playlists = Vec::new(); + let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count(); + // Downlod the sources for input in &source.inputs { let input_id = input.id; if is_input_enabled(enabled_inputs, input.enabled, input_id, &user_targets) { @@ -306,16 +307,16 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

download::get_m3u_playlist(&cfg, input, &cfg.working_dir).await, InputType::Xtream => download::get_xtream_playlist(input, &cfg.working_dir).await, }; + // @TODO optmization dont hold tv_guide in memory, persist raw and later use sax parser to extract. let (tvguide, mut tvguide_errors) = if error_list.is_empty() { download::get_xmltv(&cfg, input, &cfg.working_dir).await } else { (None, vec![]) }; - error_list.drain(..).for_each(|err| errors.push(err)); - tvguide_errors.drain(..).for_each(|err| errors.push(err)); + errors.extend(error_list.drain(..).chain(tvguide_errors.drain(..))); let input_name = match &input.name { + Some(name_val) => name_val.as_str(), None => input.url.as_str(), - Some(name_val) => name_val.as_str() }; let group_count = playlistgroups.len(); let channel_count = playlistgroups.iter() @@ -334,19 +335,8 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

, source_idx: usize, user_targets: Arc

{} - Err(mut err) => err.drain(..).for_each(|e| errors.push(e)) + Err(mut err) => errors.extend(err.drain(..)) } } } @@ -370,6 +360,22 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

InputStats { + InputStats { + name: input_name.to_string(), + input_type: input_type, + error_count: error_count, + raw_stats: PlaylistStats { + group_count, + channel_count, + }, + processed_stats: PlaylistStats { + group_count: 0, + channel_count: 0, + }, + } +} + async fn process_sources(config: Arc, user_targets: Arc) -> (Vec, Vec) { let mut handle_list = vec![]; let thread_num = config.threads; @@ -390,10 +396,8 @@ async fn process_sources(config: Arc, user_targets: Arc) let (mut res_stats, mut res_errors) = System::new().block_on(async { process_source(cfg, index, usr_trgts).await }); - res_errors.drain(..) - .for_each(|err| shared_errors.lock().unwrap().push(err)); - res_stats.drain(..) - .for_each(|stat| shared_stats.lock().unwrap().push(stat)); + shared_errors.lock().unwrap().extend(res_errors.drain(..)); + shared_stats.lock().unwrap().extend(res_stats.drain(..)); }; handles.push(thread::spawn(process)); if handles.len() >= thread_num as usize { @@ -401,10 +405,8 @@ async fn process_sources(config: Arc, user_targets: Arc) } } else { let (mut res_stats, mut res_errors) = process_source(cfg, index, usr_trgts).await; - res_errors.drain(..) - .for_each(|err| shared_errors.lock().unwrap().push(err)); - res_stats.drain(..) - .for_each(|stat| shared_stats.lock().unwrap().push(stat)); + shared_errors.lock().unwrap().extend(res_errors.drain(..)); + shared_stats.lock().unwrap().extend(res_stats.drain(..)); } } for handle in handle_list { @@ -493,12 +495,11 @@ async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>], let mut new_epg = vec![]; new_fetched_playlists.drain(..).for_each(|mut fp| { + // collect all epg_channel ids let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) .filter_map(|c| c.header.borrow().epg_channel_id.clone()).collect(); - fp.playlistgroups.drain(..).for_each(|group| { - new_playlist.push(group); - }); + new_playlist.extend(fp.playlistgroups.drain(..)); if !epg_channel_ids.is_empty() { if let Some(tv_guide) = fp.epg { debug!("found epg information for {}", &target.name); diff --git a/src/processing/xtream_parser.rs b/src/processing/xtream_parser.rs index 80dabba4a..96246bfbe 100644 --- a/src/processing/xtream_parser.rs +++ b/src/processing/xtream_parser.rs @@ -8,12 +8,11 @@ use crate::create_m3u_filter_error_result; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::ConfigInput; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamStream}; -use crate::utils::default_utils::default_as_empty_rc_str; +use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream}; -fn map_to_xtream_category(category: &Value) -> Result, M3uFilterError> { - match serde_json::from_value::>(category.to_owned()) { - Ok(category_list) => Ok(category_list), +fn map_to_xtream_category(categories: &Value) -> Result, M3uFilterError> { + match serde_json::from_value::>(categories.to_owned()) { + Ok(xtream_categories) => Ok(xtream_categories), Err(err) => { create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "Failed to process categories {}", &err) } @@ -29,6 +28,16 @@ fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: &Value) -> Resu } } +fn create_xtream_series_info_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Rc { + if episode.direct_source.is_empty() { + let ext = episode.container_extension.clone(); + let stream_base_url = format!("{url}/series/{username}/{password}/{}.{ext}", episode.id); + Rc::new(stream_base_url) + } else { + Rc::new(episode.direct_source.clone()) + } +} + pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &ConfigInput) -> Result>, M3uFilterError> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); @@ -39,33 +48,18 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: & let result: Vec = series_info.episodes.values().flatten().map(|episode| PlaylistItem { header: RefCell::new(PlaylistItemHeader { - uuid: default_as_empty_rc_str(), - id: Rc::new(episode.id.clone()), - stream_id: Rc::new(episode.id.clone()), + id: Rc::new(episode.id.to_string()), name: Rc::new(episode.title.clone()), - chno: default_as_empty_rc_str(), logo: Rc::new(episode.info.movie_image.clone()), - logo_small: default_as_empty_rc_str(), group: Rc::new(group_title.to_string()), title: Rc::new(episode.title.clone()), - parent_code: default_as_empty_rc_str(), - audio_track: default_as_empty_rc_str(), - time_shift: default_as_empty_rc_str(), - rec: default_as_empty_rc_str(), - url: if episode.direct_source.is_empty() { - let ext = episode.container_extension.clone(); - let stream_base_url = format!("{}/series/{}/{}/{}.{}", url, username, password, episode.id.as_str(), ext); - Rc::new(stream_base_url) - } else { - Rc::new(episode.direct_source.clone()) - }, - epg_channel_id: None, + url: create_xtream_series_info_url(url, username, password, episode), item_type: PlaylistItemType::Series, xtream_cluster: XtreamCluster::Series, additional_properties: episode.get_additional_properties(&series_info), - series_fetched: false, category_id: 0, input_id: input.id, + ..Default::default() }) }).collect(); if result.is_empty() { Ok(None) } else { Ok(Some(result)) } @@ -76,69 +70,61 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: & } } +fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &str, password: &str, stream: &XtreamStream) -> Rc { + if stream.direct_source.is_empty() { + let stream_base_url = match xtream_cluster { + XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()), + XtreamCluster::Video => { + let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str()); + format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext) + } + XtreamCluster::Series => + format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}", + url, username, password, &stream.get_stream_id()) + }; + Rc::new(stream_base_url) + } else { + Rc::clone(&stream.direct_source) + } +} pub(crate) fn parse_xtream(input: &ConfigInput, xtream_cluster: XtreamCluster, - category: &Value, + categories: &Value, streams: &Value) -> Result>, M3uFilterError> { - match map_to_xtream_category(category) { - Ok(mut categories) => { + match map_to_xtream_category(categories) { + Ok(mut xtream_categories) => { let input_id = input.id; let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); return match map_to_xtream_streams(xtream_cluster, streams) { - Ok(streams) => { + Ok(xtream_streams) => { let group_map: HashMap::, RefCell> = - categories.drain(..).map(|category| + xtream_categories.drain(..).map(|category| (Rc::clone(&category.category_id), RefCell::new(category)) ).collect(); - for stream in streams { + for stream in xtream_streams { if let Some(group) = group_map.get(&stream.category_id) { let mut grp = group.borrow_mut(); let category_name = &grp.category_name; let item = PlaylistItem { header: RefCell::new(PlaylistItemHeader { - uuid: default_as_empty_rc_str(), - id: Rc::new(stream.get_stream_id()), - stream_id: Rc::new(stream.get_stream_id()), + id: Rc::new(stream.get_stream_id().to_string()), name: Rc::clone(&stream.name), - chno: default_as_empty_rc_str(), logo: Rc::clone(&stream.stream_icon), - logo_small: default_as_empty_rc_str(), group: Rc::clone(category_name), title: Rc::clone(&stream.name), - parent_code: default_as_empty_rc_str(), - audio_track: default_as_empty_rc_str(), - time_shift: default_as_empty_rc_str(), - rec: default_as_empty_rc_str(), - url: if stream.direct_source.is_empty() { - let stream_base_url = match xtream_cluster { - XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()), - XtreamCluster::Video => { - let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str()); - format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext) - } - XtreamCluster::Series => - format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}", - url, username, password, &stream.get_stream_id()) - }; - Rc::new(stream_base_url) - } else { - Rc::clone(&stream.direct_source) - }, + url: create_xtream_url(xtream_cluster, url, username, password, &stream), epg_channel_id: stream.epg_channel_id.clone(), - item_type: match xtream_cluster { - XtreamCluster::Live => PlaylistItemType::Live, - XtreamCluster::Video => PlaylistItemType::Movie, - XtreamCluster::Series => PlaylistItemType::SeriesInfo, - }, + item_type: PlaylistItemType::from(xtream_cluster), xtream_cluster, additional_properties: stream.get_additional_properties(), series_fetched: false, category_id: 0, input_id, + ..Default::default() }), }; grp.add(item); @@ -160,4 +146,5 @@ pub(crate) fn parse_xtream(input: &ConfigInput, } Err(err) => Err(err) } -} \ No newline at end of file +} + diff --git a/src/repository/bplustree.rs b/src/repository/bplustree.rs deleted file mode 100644 index 03127ae81..000000000 --- a/src/repository/bplustree.rs +++ /dev/null @@ -1,181 +0,0 @@ -use std::fs::File; -use std::io::{self}; -use flate2::Compression; -use flate2::write::{GzEncoder}; -use flate2::read::{GzDecoder}; - -use serde::{Deserialize, Serialize}; - -const T: usize = 3; // Minimum degree (T), meaning each node can contain at most 2*T - 1 keys - -#[derive(Serialize, Deserialize, Debug, Clone)] -struct KeyValue { - key: K, - value: V, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -struct BPlusTreeNode { - keys: Vec, - children: Vec>>, - is_leaf: bool, - values: Vec>, // only used in leaf nodes -} - -impl BPlusTreeNode -where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, - V: Serialize + for<'de> Deserialize<'de> + Clone, -{ - fn new(is_leaf: bool) -> Self { - BPlusTreeNode { - keys: vec![], - children: vec![], - is_leaf, - values: vec![], - } - } - - fn insert_non_full(&mut self, key: K, value: V) { - let pos = self.keys.binary_search(&key).unwrap_or_else(|pos| pos); - - if self.is_leaf { - self.keys.insert(pos, key); - self.values.insert(pos, Some(value)); - } else { - let mut idx = pos; - if self.children[pos].as_mut().unwrap().keys.len() == 2 * T - 1 { - self.split_child(pos); - if key > self.keys[pos] { - idx = pos + 1; - } - } - self.children[idx].as_mut().unwrap().insert_non_full(key, value); - } - } - - fn split_child(&mut self, pos: usize) { - let t = T - 1; - let mut new_node = BPlusTreeNode::new(self.children[pos].as_ref().unwrap().is_leaf); - let mut old_node = self.children[pos].take().unwrap(); - - self.keys.insert(pos, old_node.keys.remove(t)); - self.children.insert(pos + 1, Some(new_node.clone())); - - if old_node.is_leaf { - new_node.keys = old_node.keys.split_off(t); - new_node.values = old_node.values.split_off(t); - } else { - new_node.keys = old_node.keys.split_off(t + 1); - new_node.children = old_node.children.split_off(t + 1); - } - - self.children[pos] = Some(old_node); - self.children[pos + 1] = Some(new_node); - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -struct BPlusTree { - root: BPlusTreeNode, -} - -impl BPlusTree -where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, - V: Serialize + for<'de> Deserialize<'de> + Clone, -{ - pub(crate) fn new() -> Self { - BPlusTree { - root: BPlusTreeNode::new(true), - } - } - - pub(crate) fn insert(&mut self, key: K, value: V) { - if self.root.keys.len() == 2 * T - 1 { - let mut new_root = BPlusTreeNode::new(false); - new_root.children.push(Some(self.root.clone())); - new_root.split_child(0); - self.root = new_root; - } - self.root.insert_non_full(key, value); - } - - pub(crate) fn query(&self, key: &K) -> Option { - let mut node = &self.root; - while !node.is_leaf { - let pos = match node.keys.binary_search(key) { - Ok(pos) => return node.values[pos].clone(), - Err(pos) => pos, - }; - node = node.children[pos].as_ref().unwrap(); - } - - match node.keys.binary_search(key) { - Ok(pos) => node.values[pos].clone(), - Err(_) => None, - } - } - - pub(crate) fn serialize_to_file(&self, filename: &str) -> io::Result<()> { - let file = File::create(filename)?; - let encoder = GzEncoder::new(file, Compression::default()); - match bincode::serialize_into(encoder, &self) { - Ok(()) => Ok(()), - Err(e) => { - println!("Failed to write bplustree to disk {e}"); - Ok(()) - } - } - } - - // If file exists the file is deserialized, otherwise an empty tree is returned - pub(crate) fn deserialize_from_file(filename: &str) -> Self { - match File::open(filename) { - Ok(file) => { - let decoder = GzDecoder::new(file); - let tree: BPlusTree = bincode::deserialize_from(decoder).unwrap(); - tree - } - Err(_) => BPlusTree::new() - } - } -} - -#[cfg(test)] -mod tests { - use std::io; - use serde::{Deserialize, Serialize}; - use crate::repository::bplustree::{BPlusTree}; - - - // Example usage with a simple struct - #[derive(Serialize, Deserialize, Debug, Clone)] - struct Value { - id: u32, - data: String, - } - - #[test] - fn insert_test() -> io::Result<()> { - let mut tree = BPlusTree::new(); - tree.insert("abc".to_string(), Value { id: 16, data: "one".to_string() }); - tree.insert("def".to_string(), Value { id: 32, data: "two".to_string() }); - tree.insert("ghi".to_string(), Value { id: 64, data: "three".to_string() }); - - // Serialize the tree to a file - tree.serialize_to_file("/tmp/tree.bin")?; - - // Deserialize the tree from the file - let tree = BPlusTree::::deserialize_from_file("/tmp/tree.bin"); - - // Query the tree - if let Some(value) = tree.query(&("ghi".to_string())) { - println!("Found: {:?}", value); - } else { - println!("Not found"); - } - - Ok(()) - } -} \ No newline at end of file diff --git a/src/repository/epg_repository.rs b/src/repository/epg_repository.rs index 211930b59..528497d71 100644 --- a/src/repository/epg_repository.rs +++ b/src/repository/epg_repository.rs @@ -42,7 +42,7 @@ fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path) -> Result<(), M Ok(()) } -pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, epg: Option<&Epg>, output: &TargetOutput) -> Result<(), M3uFilterError> { +pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, target_path: &Path, epg: Option<&Epg>, output: &TargetOutput) -> Result<(), M3uFilterError> { if let Some(epg_data) = epg { match &output.target { TargetType::M3u => { @@ -51,7 +51,7 @@ pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, epg: Option<&Epg>, M3uFilterErrorKind::Notify, format!("write epg for target {} failed: No filename set", target.name))); } - if let Some(path) = m3u_get_epg_file_path(cfg, target) { + if let Some(path) = m3u_get_epg_file_path(target_path) { if log_enabled!(Level::Debug) { debug!("writing m3u epg to {}", path.to_str().unwrap_or("?")); } diff --git a/src/repository/index_record.rs b/src/repository/index_record.rs index 1bb0e7808..c4cf37ac1 100644 --- a/src/repository/index_record.rs +++ b/src/repository/index_record.rs @@ -39,23 +39,6 @@ impl IndexRecord { } } - // pub fn from_bytes(bytes: &[u8], cursor: &mut usize) -> Result { - // if let Ok(index_bytes) = bytes[*cursor..*cursor + 4].try_into() { - // *cursor += 4; - // if let Ok(size_bytes) = bytes[*cursor..*cursor + 2].try_into() { - // *cursor += 2; - // let index = u32::from_le_bytes(index_bytes); - // let size = u16::from_le_bytes(size_bytes); - // return Ok(IndexRecord { index, size }); - // } - // } - // Err(Error::new(ErrorKind::Other, "Failed to read index")) - // } - - // pub fn as_bytes(&self) -> [u8; 6] { - // IndexRecord::to_bytes(self.index, self.size) - // } - pub fn to_bytes(left: u32, right: u32) -> [u8; 8] { let left_bytes: [u8; 4] = left.to_le_bytes(); let right_bytes: [u8; 4] = right.to_le_bytes(); diff --git a/src/repository/indexed_document_writer.rs b/src/repository/indexed_document_writer.rs index 289bdca7f..b05d7439f 100644 --- a/src/repository/indexed_document_writer.rs +++ b/src/repository/indexed_document_writer.rs @@ -49,7 +49,7 @@ impl IndexedDocumentWriter { Self::new_with_mode(main_path, index_path, true) } - pub fn write_doc(&mut self, doc: &T) -> Result<(u32, u32), Error> + pub fn write_doc(&mut self, doc_id: u32, doc: &T) -> Result<(u32, u32), Error> where T: ?Sized + serde::Serialize { let current_main_index = self.main_offset; @@ -58,7 +58,7 @@ impl IndexedDocumentWriter { match file_utils::check_write(&self.main_file.write_all(&encoded)) { Ok(()) => { let bytes_written = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; - let combined_bytes = IndexRecord::to_bytes(self.main_offset, bytes_written); + let combined_bytes = IndexRecord::to_bytes(doc_id, bytes_written); if let Err(err) = file_utils::check_write(&self.index_file.write_all(&combined_bytes)) { return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.index_path.to_str().unwrap(), err))); } diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 4f96daa97..214c73b4d 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -1,7 +1,6 @@ use std::fs::File; use std::io::{BufWriter, Error, ErrorKind, Write}; use std::path::{Path, PathBuf}; -use std::rc::Rc; use log::error; @@ -14,6 +13,7 @@ use crate::model::playlist::{M3uPlaylistItem, PlaylistGroup, PlaylistItem, Playl use crate::repository::index_record::IndexRecord; use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; use crate::repository::indexed_document_writer::IndexedDocumentWriter; +use crate::repository::storage::{ensure_target_storage_path}; use crate::utils::file_utils; macro_rules! cant_write_result { @@ -22,12 +22,12 @@ macro_rules! cant_write_result { } } -fn m3u_get_base_file_path(cfg: &Config, target: &ConfigTarget) -> Option { - file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(format!("m3u_{}.db", target.name.replace(' ', "_").as_str())))) +fn m3u_get_base_file_path(target_path: &Path) -> Option { + Some(target_path.join(PathBuf::from("m3u.db"))) } -pub(crate) fn m3u_get_file_paths(cfg: &Config, target: &ConfigTarget) -> Option<(PathBuf, PathBuf)> { - match m3u_get_base_file_path(cfg, target) { +pub(crate) fn m3u_get_file_paths(target_path: &Path) -> Option<(PathBuf, PathBuf)> { + match m3u_get_base_file_path(target_path) { Some(m3u_path) => { let extension = m3u_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or(""))); let index_path = m3u_path.with_extension(format!("{}idx", &extension.unwrap_or(String::new()))); @@ -37,44 +37,47 @@ pub(crate) fn m3u_get_file_paths(cfg: &Config, target: &ConfigTarget) -> Option< } } -pub(crate) fn m3u_get_epg_file_path(cfg: &Config, target: &ConfigTarget) -> Option { - m3u_get_base_file_path(cfg, target) +pub(crate) fn m3u_get_epg_file_path(target_path: &Path) -> Option { + m3u_get_base_file_path(target_path) .map(|path| file_utils::add_prefix_to_filename(&path, "epg_", Some("xml"))) } -pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { +fn persist_m3u_playlist_as_text(target: &ConfigTarget, cfg: &Config, m3u_playlist: &Vec) { + if let Some(filename) = target.get_m3u_filename() { + if let Some(m3u_filename) = file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) { + match File::create(&m3u_filename) { + Ok(file) => { + let mut buf_writer = BufWriter::new(file); + let _ = buf_writer.write("#EXTM3U\n".as_bytes()); + for m3u in m3u_playlist { + let _ = buf_writer.write(m3u.to_m3u(target, None).as_bytes()); + let _ = buf_writer.write("\n".as_bytes()); + } + } + Err(_) => { + error!("Can't write m3u plain playlist {}", &m3u_filename.to_str().unwrap()); + } + } + } + } +} + +pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, target_path: &Path, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { if !new_playlist.is_empty() { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(cfg, target) { + if let Some((m3u_path, idx_path)) = m3u_get_file_paths(target_path) { let m3u_playlist = new_playlist.iter() .flat_map(|pg| &pg.channels) .filter(|&pli| pli.header.borrow().item_type != PlaylistItemType::SeriesInfo) .map(PlaylistItem::to_m3u).collect::>(); - if let Some(filename) = target.get_m3u_filename() { - if let Some(m3u_filename) = file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) { - match File::create(&m3u_filename) { - Ok(file) => { - let mut buf_writer = BufWriter::new(file); - let _ = buf_writer.write("#EXTM3U\n".as_bytes()); - for m3u in &m3u_playlist { - let _ = buf_writer.write(m3u.to_m3u(target, None).as_bytes()); - let _ = buf_writer.write("\n".as_bytes()); - } - } - Err(_) => { - error!("Can't write m3u plain playlist {}", &m3u_filename.to_str().unwrap()); - } - } - } - } + persist_m3u_playlist_as_text(target, cfg, &m3u_playlist); + match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) { Ok(mut writer) => { - let mut stream_id: u32 = 1; - for mut m3u in m3u_playlist { - m3u.stream_id = Rc::new(stream_id.to_string()); - match writer.write_doc(&m3u) { - Ok(_) => stream_id += 1, + for m3u in m3u_playlist { + match writer.write_doc(m3u.virtual_id, &m3u) { + Ok(_) => {}, Err(err) => return cant_write_result!(&m3u_path, err) } } @@ -87,36 +90,43 @@ pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, new_playli } pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, user: &ProxyUserCredentials) -> Option { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(cfg, target) { - match IndexedDocumentReader::::new(&m3u_path, &idx_path) { - Ok(mut reader) => { - let server_info = get_user_server_info(cfg, user); - let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); - let mut result = vec![]; - result.push("#EXTM3U".to_string()); - for m3u_pli in reader.by_ref() { - match user.proxy { - ProxyType::Reverse => { - let stream_id = Rc::clone(&m3u_pli.stream_id); - result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{stream_id}").as_str()))); - } - ProxyType::Redirect => { - result.push(m3u_pli.to_m3u(target, None)); + match ensure_target_storage_path(cfg, target.name.as_str()) { + Ok(target_path) => { + if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) { + match IndexedDocumentReader::::new(&m3u_path, &idx_path) { + Ok(mut reader) => { + let server_info = get_user_server_info(cfg, user); + let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); + let mut result = vec![]; + result.push("#EXTM3U".to_string()); + for m3u_pli in reader.by_ref() { + match user.proxy { + ProxyType::Reverse => { + let stream_id = m3u_pli.virtual_id; + result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{stream_id}").as_str()))); + } + ProxyType::Redirect => { + result.push(m3u_pli.to_m3u(target, None)); + } + } + }; + if reader.by_ref().has_error() { + error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap()); + } else { + return Some(result.join("\n")); } } - }; - if reader.by_ref().has_error() { - error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap()); - } else { - return Some(result.join("\n")); + Err(err) => { + error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err); + } } + } else { + error!("Could not open files for target {}", &target.name); } - Err(err) => { - error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err); - } + }, + Err(err) => { + error!("Could not find storage path for target {} - {}", target.name.as_str(), err); } - } else { - error!("Could not open files for target {}", &target.name); } None } diff --git a/src/repository/mod.rs b/src/repository/mod.rs index a1f08200b..8ca12a2fa 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -2,10 +2,10 @@ pub(crate) mod playlist_repository; pub(crate) mod m3u_repository; pub(crate) mod xtream_repository; pub(crate) mod epg_repository; -pub(crate)mod kodi_repository; - -mod bplustree; +pub(crate) mod kodi_repository; +pub(crate) mod storage; mod index_record; mod indexed_document_writer; -mod indexed_document_reader; \ No newline at end of file +mod indexed_document_reader; +mod target_id_mapping_record; \ No newline at end of file diff --git a/src/repository/playlist_repository.rs b/src/repository/playlist_repository.rs index 3a820f793..51662fc53 100644 --- a/src/repository/playlist_repository.rs +++ b/src/repository/playlist_repository.rs @@ -1,31 +1,59 @@ -use crate::m3u_filter_error::M3uFilterError; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget, TargetType}; use crate::model::playlist::PlaylistGroup; use crate::model::xmltv::Epg; use crate::repository::epg_repository::epg_write; use crate::repository::kodi_repository::kodi_write_strm_playlist; use crate::repository::m3u_repository::m3u_write_playlist; +use crate::repository::storage::{ensure_target_storage_path, get_target_id_mapping_file}; +use crate::repository::target_id_mapping_record::TargetIdMapping; use crate::repository::xtream_repository::xtream_write_playlist; pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg>, target: &ConfigTarget, cfg: &Config) -> Result<(), Vec> { let mut errors = vec![]; - for output in &target.output { - match match output.target { - TargetType::M3u => m3u_write_playlist(target, cfg, playlist), - TargetType::Xtream => xtream_write_playlist(target, cfg, playlist), - TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), - } { - Ok(()) => { - if !playlist.is_empty() { - match epg_write(target, cfg, epg, output) { - Ok(()) => {} - Err(err) => errors.push(err) + + // TODO get previous virtual-ids and match them to the playlist items + match ensure_target_storage_path(cfg, target.name.as_str()) { + Ok(target_path) => { + let mut target_id_mapping = TargetIdMapping::from_path(&get_target_id_mapping_file(&target_path)); + for group in &mut *playlist { + for channel in &group.channels { + let mut header = channel.header.borrow_mut(); + match header.id.parse::() { + Ok(provider_id) => { + let uuid = header.get_uuid(); + header.virtual_id = target_id_mapping.insert_entry(provider_id, **uuid); + } + Err(err) => { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } } } } - Err(err) => errors.push(err) + + for output in &target.output { + match match output.target { + TargetType::M3u => m3u_write_playlist(target, cfg, &target_path, playlist), + TargetType::Xtream => xtream_write_playlist(target, cfg, &target_path, playlist), + TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), + } { + Ok(()) => { + if let Err(err) = target_id_mapping.to_path(&target_path) { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } + if !playlist.is_empty() { + match epg_write(target, cfg, &target_path, epg, output) { + Ok(()) => {} + Err(err) => errors.push(err) + } + } + } + Err(err) => errors.push(err) + } + } } + Err(err) => errors.push(err), } if errors.is_empty() { Ok(()) } else { Err(errors) } diff --git a/src/repository/storage.rs b/src/repository/storage.rs new file mode 100644 index 000000000..62e899703 --- /dev/null +++ b/src/repository/storage.rs @@ -0,0 +1,30 @@ +use std::path::{Path, PathBuf}; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; +use crate::model::config::Config; +use crate::utils::file_utils; + +pub(crate) fn hash_string(url: &str) -> [u8; 32] { + let hash = blake3::hash(url.as_bytes()); + hash.into() // Konvertiere den Hash in ein Array mit fester Größe +} + +pub(crate) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { + target_path.join(PathBuf::from("id_mapping.db")) +} + +pub(crate) fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result { + if let Some(path) = get_target_storage_path(cfg, target_name) { + if std::fs::create_dir_all(&path).is_err() { + let msg = format!("Failed to save target data, can't create directory {}", &path.to_str().unwrap()); + return Err(M3uFilterError::new(M3uFilterErrorKind::Notify, msg)); + } + Ok(path) + } else { + let msg = format!("Failed to save target data, can't create directory for target {target_name}"); + Err(M3uFilterError::new(M3uFilterErrorKind::Notify, msg)) + } +} + +pub(crate) fn get_target_storage_path(cfg: &Config, target_name: &str) -> Option { + file_utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(' ', "_")))) +} diff --git a/src/repository/target_id_mapping_record.rs b/src/repository/target_id_mapping_record.rs new file mode 100644 index 000000000..7e638ddae --- /dev/null +++ b/src/repository/target_id_mapping_record.rs @@ -0,0 +1,141 @@ +use std::cmp::max; +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{Error, Read, Seek, SeekFrom, Write}; +use std::path::Path; +use std::rc::Rc; + +use crate::utils::file_utils; + +/** +This file contains the provider id, the virtual id, and an uuid + + */ +pub(in crate::repository) struct TargetIdMappingRecord { + pub provider_id: u32, + pub virtual_id: u32, + pub uuid: [u8; 32], +} + +impl TargetIdMappingRecord { + fn to_bytes(&self) -> [u8; 40] { + let provider_id_bytes: [u8; 4] = self.provider_id.to_le_bytes(); + let virtual_id_bytes: [u8; 4] = self.virtual_id.to_le_bytes(); + let mut combined_bytes: [u8; 40] = [0; 40]; + combined_bytes[..4].copy_from_slice(&provider_id_bytes); + combined_bytes[4..].copy_from_slice(&virtual_id_bytes); + combined_bytes[8..].copy_from_slice(&self.uuid); + combined_bytes + } +} + +pub(in crate::repository) struct TargetIdMapping { + dirty: bool, + virtual_id_counter: u32, + by_provider_id: BTreeMap>, + by_virtual_id: BTreeMap>, + by_uuid: BTreeMap<[u8; 32], Rc>, + records: Vec>, +} + +impl TargetIdMapping { + pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32]) -> u32 { + self.dirty = true; + self.virtual_id_counter += 1; + self.insert(TargetIdMappingRecord { provider_id, virtual_id: self.virtual_id_counter, uuid }); + self.virtual_id_counter + } + + fn new(records: Vec>) -> Self { + let mut by_provider_id: BTreeMap> = BTreeMap::new(); + let mut by_virtual_id: BTreeMap> = BTreeMap::new(); + let mut by_uuid: BTreeMap<[u8; 32], Rc> = BTreeMap::new(); + let mut virtual_id_counter: u32 = 0; + for record in &records { + by_provider_id.insert(record.provider_id, Rc::clone(record)); + by_virtual_id.insert(record.virtual_id, Rc::clone(record)); + by_uuid.insert(record.uuid, Rc::clone(record)); + virtual_id_counter = max(record.virtual_id, virtual_id_counter); + } + TargetIdMapping { + dirty: false, + virtual_id_counter, + by_provider_id, + by_virtual_id, + by_uuid, + records, + } + } + + fn insert(&mut self, record: TargetIdMappingRecord) { + let provider_id = record.provider_id; + let virtual_id = record.virtual_id; + let uuid = record.uuid; + let shared_record = Rc::new(record); + self.by_provider_id.insert(provider_id, Rc::clone(&shared_record)); + self.by_virtual_id.insert(virtual_id, Rc::clone(&shared_record)); + self.by_uuid.insert(uuid, Rc::clone(&shared_record)); + self.records.push(shared_record); + } + + fn get_by_provider_id(&self, provider_id: u32) -> Option<&Rc> { + self.by_provider_id.get(&provider_id) + } + + fn get_by_virtual_id(&self, virtual_id: u32) -> Option<&Rc> { + self.by_virtual_id.get(&virtual_id) + } + + fn get_by_uuid(&self, uuid: &[u8; 32]) -> Option<&Rc> { + self.by_uuid.get(uuid) + } + + pub fn to_file(&mut self, file: &mut File) -> Result<(), Error> { + for record in &self.records { + let bytes = record.to_bytes(); + if let Err(err) = file.write_all(&bytes) { + return Err(err); + } + } + self.dirty = false; + Ok(()) + } + + pub fn from_path(path: &Path) -> Self { + match file_utils::open_file_append(path, false) { + Ok(mut file) => TargetIdMapping::from_file(&mut file), + _ => TargetIdMapping::new(vec![]) + } + } + + pub fn to_path(&mut self, path: &Path) -> Result<(), Error> { + match file_utils::open_file_append(path, false) { + Ok(mut file) => self.to_file(&mut file), + Err(err) => Err(err) + } + } + + pub fn from_file(file: &mut File) -> Self { + let mut records = vec![]; + if let Ok(_) = file.seek(SeekFrom::Start(0)) { + loop { + let mut provider_id_bytes = [0u8; 4]; + let mut virtual_id_bytes = [0u8; 4]; + let mut uuid = [0u8; 32]; + if let Err(_) = file.read_exact(&mut provider_id_bytes) { + break; + } + if let Err(_) = file.read_exact(&mut virtual_id_bytes) { + break; + } + if let Err(_) = file.read_exact(&mut uuid) { + break; + } + let provider_id = u32::from_le_bytes(provider_id_bytes); + let virtual_id = u32::from_le_bytes(virtual_id_bytes); + records.push(Rc::new(TargetIdMappingRecord { provider_id, virtual_id, uuid })); + } + } + TargetIdMapping::new(records) + } +} \ No newline at end of file diff --git a/src/repository/xtream_repository.rs b/src/repository/xtream_repository.rs index a3c362712..7fbed66c8 100644 --- a/src/repository/xtream_repository.rs +++ b/src/repository/xtream_repository.rs @@ -17,6 +17,7 @@ use crate::processing::m3u_parser::extract_id_from_url; use crate::repository::index_record::IndexRecord; use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; use crate::repository::indexed_document_writer::IndexedDocumentWriter; +use crate::repository::storage::get_target_storage_path; use crate::utils::file_utils; use crate::utils::json_utils::{json_iter_array, json_write_documents_to_file}; @@ -100,8 +101,8 @@ fn write_playlist_to_file(storage_path: &Path, stream_id: &mut u32, cluster: Xtr Ok(mut writer) => { for pli in playlist.iter_mut() { if let Ok(mut xtream) = pli.to_xtream() { - xtream.stream_id = *stream_id; - match writer.write_doc(&xtream) { + xtream.virtual_id = *stream_id; + match writer.write_doc(xtream.virtual_id, &xtream) { Ok(_) => *stream_id += 1, Err(err) => return cant_write_result!(&xtream_path, err) } @@ -193,7 +194,10 @@ fn load_old_category_ids(path: &Path) -> (u32, HashMap) { } pub(crate) fn xtream_get_storage_path(cfg: &Config, target_name: &str) -> Option { - file_utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(' ', "_")))) + match get_target_storage_path(cfg, target_name) { + Some(target_path) => Some(target_path.join(std::path::PathBuf::from("xtream"))), + None => None, + } } pub(crate) fn xtream_get_epg_file_path(path: &Path) -> PathBuf { @@ -211,7 +215,7 @@ pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: XtreamCluster) (xtream_path, index_path) } -pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut [PlaylistGroup]) -> Result<(), M3uFilterError> { +pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, target_path: &Path, playlist: &mut [PlaylistGroup]) -> Result<(), M3uFilterError> { match ensure_xtream_storage_path(cfg, target.name.replace(' ', "_").as_str()) { Ok(path) => { let mut cat_live_col = vec![]; @@ -221,7 +225,7 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis let mut series_col = vec![]; let mut vod_col = vec![]; let mut errors = Vec::new(); - +// TODO.EUZU old catgeory ids // preserve category_ids let (max_cat_id, existing_cat_ids) = load_old_category_ids(&path); let mut cat_id_counter = max_cat_id; @@ -473,7 +477,7 @@ pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { return match IndexedDocumentWriter::new_append(info_path.clone(), idx_path) { Ok(mut writer) => { - match writer.write_doc(content) { + match writer.write_doc(series_id, content) { Ok((_, index_offset)) => { let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path); IndexRecord::to_file(&series_id_index_mapping_path, series_id, index_offset, true)?; diff --git a/src/test.rs b/src/test.rs index 4cd838d24..cd7de9006 100644 --- a/src/test.rs +++ b/src/test.rs @@ -1,35 +1,14 @@ #[cfg(test)] mod tests { use std::cell::RefCell; - use std::rc::Rc; use regex::Regex; use crate::filter::{get_filter, MockValueProcessor, ValueProvider}; - use crate::model::playlist::{PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; + use crate::model::playlist::{PlaylistItem, PlaylistItemHeader}; fn create_mock_pli(name: &str, group: &str) -> PlaylistItem { PlaylistItem { header: RefCell::new(PlaylistItemHeader { - uuid: Rc::new("".to_string()), - stream_id: Rc::new("".to_string()), - id: Rc::new("".to_string()), - name: Rc::new(name.to_string()), - chno: Rc::new("".to_string()), - logo: Rc::new("".to_string()), - logo_small: Rc::new("".to_string()), - group: Rc::new(group.to_string()), - title: Rc::new("".to_string()), - parent_code: Rc::new("".to_string()), - audio_track: Rc::new("".to_string()), - time_shift: Rc::new("".to_string()), - rec: Rc::new("".to_string()), - url: Rc::new("".to_string()), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Live, - additional_properties: None, - item_type: PlaylistItemType::Live, - series_fetched: false, - category_id: 0, - input_id: 0, + ..Default::default() }) } } diff --git a/src/utils/default_utils.rs b/src/utils/default_utils.rs index 758436c94..e51dbec7f 100644 --- a/src/utils/default_utils.rs +++ b/src/utils/default_utils.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::rc::Rc; use crate::model::config::ProcessingOrder; -use crate::model::playlist::{PlaylistItemType, XtreamCluster}; pub(crate) fn default_as_true() -> bool { true } @@ -23,8 +22,6 @@ pub(crate) fn default_as_empty_list() -> Vec { vec![] } pub(crate) fn default_as_two_u16() -> u16 { 2 } -pub(crate) fn default_playlist_item_type() -> PlaylistItemType { PlaylistItemType::Live } pub(crate) fn default_as_zero_u32() -> u32 { 0 } pub(crate) fn default_as_zero_u16() -> u16 { 0 } -pub(crate) fn default_stream_cluster() -> XtreamCluster { XtreamCluster::Live } diff --git a/src/utils/download.rs b/src/utils/download.rs index 9a0bf2358..a5ddff2de 100644 --- a/src/utils/download.rs +++ b/src/utils/download.rs @@ -30,8 +30,7 @@ pub(crate) async fn get_m3u_playlist(cfg: &Config, input: &ConfigInput, working_ let persist_file_path = prepare_file_path(input.persist.as_ref(), working_dir, ""); match request_utils::get_input_text_content(input, working_dir, &url, persist_file_path).await { Ok(text) => { - let lines = text.lines().map(String::from).collect::>(); - (m3u_parser::parse_m3u(cfg, input, &lines), vec![]) + (m3u_parser::parse_m3u(cfg, input, text.lines()), vec![]) } Err(err) => (vec![], vec![err]) } @@ -57,7 +56,7 @@ pub(crate) async fn get_xtream_playlist_series<'a>(fpl: &mut FetchedPlaylist<'a> match parse_xtream_series_info(&series_content, pli.header.borrow().group.as_str(), input) { Ok(series_info) => { if let Some(mut series) = series_info { - series.drain(..).for_each(|item| group_series.push(item)); + group_series.extend(series.drain(..)); } } Err(err) => errors.push(err), @@ -108,7 +107,7 @@ const ACTIONS: [(XtreamCluster, &str, &str); 3] = [ (XtreamCluster::Series, "get_series_categories", "get_series")]; pub(crate) async fn get_xtream_playlist(input: &ConfigInput, working_dir: &String) -> (Vec, Vec) { - let mut playlist: Vec = Vec::new(); + let mut playlist_groups: Vec = Vec::new(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); let base_url = format!("{}/player_api.php?username={}&password={}", input.url, username, password); @@ -123,35 +122,37 @@ pub(crate) async fn get_xtream_playlist(input: &ConfigInput, working_dir: &Strin let category_file_path = prepare_file_path(input.persist.as_ref(), working_dir, format!("{category}_").as_str()); let stream_file_path = prepare_file_path(input.persist.as_ref(), working_dir, format!("{stream}_").as_str()); - match request_utils::get_input_json_content(input, category_url.as_str(), category_file_path).await { - Ok(category_content) => { - match request_utils::get_input_json_content(input, stream_url.as_str(), stream_file_path).await { - Ok(stream_content) => { - match xtream_parser::parse_xtream(input, - *xtream_cluster, - &category_content, - &stream_content) { - Ok(sub_playlist_opt) => { - if let Some(mut sub_playlist) = sub_playlist_opt { - sub_playlist.drain(..).for_each(|group| playlist.push(group)); - } - } - Err(err) => errors.push(err) + match futures::join!( + request_utils::get_input_json_content(input, category_url.as_str(), category_file_path), + request_utils::get_input_json_content(input, stream_url.as_str(), stream_file_path) + ) { + (Ok(category_content), Ok(stream_content)) => { + match xtream_parser::parse_xtream(input, + *xtream_cluster, + &category_content, + &stream_content) { + Ok(sub_playlist_parsed) => { + if let Some(mut xtream_sub_playlist) = sub_playlist_parsed { + playlist_groups.extend(xtream_sub_playlist.drain(..)); } } Err(err) => errors.push(err) } - } - Err(err) => errors.push(err) + }, + (Err(err1), Err(err2)) => { + errors.extend([err1, err2]); + }, + (Err(err), _) => errors.push(err), + (_, Err(err)) => errors.push(err), } } } - playlist.sort_by(|a, b| a.title.partial_cmp(&b.title).unwrap_or(Ordering::Greater)); + playlist_groups.sort_by(|a, b| a.title.partial_cmp(&b.title).unwrap_or(Ordering::Greater)); - for (grp_id, plg) in (1_u32..).zip(playlist.iter_mut()) { + for (grp_id, plg) in (1_u32..).zip(playlist_groups.iter_mut()) { plg.id = grp_id; } - (playlist, errors) + (playlist_groups, errors) } pub(crate) async fn get_xmltv(_cfg: &Config, input: &ConfigInput, working_dir: &String) -> (Option, Vec) { diff --git a/src/utils/request_utils.rs b/src/utils/request_utils.rs index 10b282c86..9d13c5c1a 100644 --- a/src/utils/request_utils.rs +++ b/src/utils/request_utils.rs @@ -232,15 +232,15 @@ pub(crate) async fn get_input_json_content(input: &ConfigInput, url: &str, persi Err(e) => create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "cant download input url: {url} => {}", e) } } - -pub(crate) fn get_base_url(url: &str) -> Option { - if let Some((scheme_end, rest)) = url.split_once("://") { - let scheme = scheme_end; - if let Some(authority_end) = rest.find('/') { - let authority = &rest[..authority_end]; - return Some(format!("{}://{}", scheme, authority)); - } - return Some(format!("{}://{}", scheme, rest)); - } - None -} +// +// pub(crate) fn get_base_url(url: &str) -> Option { +// if let Some((scheme_end, rest)) = url.split_once("://") { +// let scheme = scheme_end; +// if let Some(authority_end) = rest.find('/') { +// let authority = &rest[..authority_end]; +// return Some(format!("{}://{}", scheme, authority)); +// } +// return Some(format!("{}://{}", scheme, rest)); +// } +// None +// } From f1ef173fc3bfa5bb4258e8e4168833c1f3cd1779 Mon Sep 17 00:00:00 2001 From: euzu Date: Fri, 25 Oct 2024 18:26:25 +0200 Subject: [PATCH 02/11] virtual id refactoring --- src/api/xtream_api.rs | 52 ++-- src/filter.rs | 6 +- src/model/playlist.rs | 91 ++++++- src/processing/m3u_parser.rs | 2 +- src/repository/index_record.rs | 53 ---- src/repository/indexed_document_reader.rs | 90 ++++--- src/repository/indexed_document_writer.rs | 36 ++- src/repository/m3u_repository.rs | 3 +- src/repository/mod.rs | 3 +- src/repository/playlist_repository.rs | 22 +- src/repository/target_id_mapping_record.rs | 114 ++++++--- src/repository/xtream_repository.rs | 282 ++++++--------------- test/rest-api.http | 4 +- 13 files changed, 358 insertions(+), 400 deletions(-) delete mode 100644 src/repository/index_record.rs diff --git a/src/api/xtream_api.rs b/src/api/xtream_api.rs index 12fb38130..e94b0da53 100644 --- a/src/api/xtream_api.rs +++ b/src/api/xtream_api.rs @@ -17,8 +17,10 @@ use crate::api::api_utils::{get_user_server_info, get_user_target, get_user_targ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigInput, ConfigTarget}; use crate::model::config::TargetType; -use crate::model::playlist::{XtreamCluster, XtreamPlaylistItem}; +use crate::model::playlist::{PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; +use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path, hash_string}; +use crate::repository::target_id_mapping_record::TargetIdMapping; use crate::repository::xtream_repository; use crate::utils::{json_utils, request_utils}; @@ -161,12 +163,12 @@ async fn xtream_player_api_stream( let target_name = &target.name; if target.has_output(&TargetType::Xtream) { let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id); - let req_stream_id: u32 = match FromStr::from_str(action_stream_id.trim()) { + let virtual_id: u32 = match FromStr::from_str(action_stream_id.trim()) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().finish() }; - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, None) { + if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None) { let input_id: u16 = pli.input_id; if let Some(input) = app_state.config.get_input_by_id(input_id) { let mut query_path = if stream_req.action_path.is_empty() { String::new() } else { format!("{}/", stream_req.action_path) }; @@ -178,12 +180,12 @@ async fn xtream_player_api_stream( } return stream_response(&stream_url, req, Some(input)).await; } - error!("Cant find stream url for target {target_name}, context {}, stream_id {req_stream_id}", stream_req.context); + error!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context); } else { - error!("Cant find input for target {target_name}, context {}, stream_id {req_stream_id}", stream_req.context); + error!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context); } } else { - error!("Failed to read xtream item for stream id {}", req_stream_id); + error!("Failed to read xtream item for stream id {}", virtual_id); } } else { debug!("Target has no xtream output {}", target_name); @@ -281,10 +283,11 @@ fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.virtual_id))) } -fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result { +fn get_and_write_xtream_series_info(config: &Config, target: &ConfigTarget, pli_series_info: &XtreamPlaylistItem, content: &str) -> Result { if let Ok(mut doc) = serde_json::from_str::(content) { - let mut new_id_to_provider_id_mapping: Vec<(u32, u32)> = Vec::new(); - if let Some(mut new_id) = xtream_repository::xtream_get_max_series_info_episode_id(config, target.name.replace(' ', "_").as_str()) { + if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { + let mut target_id_mapping = TargetIdMapping::from_path(&get_target_id_mapping_file(&target_path)); + if let Some(episodes) = doc.get_mut("episodes") { if let Some(episodes_map) = episodes.as_object_mut() { let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); @@ -295,9 +298,10 @@ fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPl if let Some(episode) = entry.as_object_mut() { if let Some(episode_id) = episode.get("id") { if let Ok(provider_id) = episode_id.as_str().unwrap().parse::() { - new_id += 1; - new_id_to_provider_id_mapping.push((new_id, provider_id)); - episode.insert("id".to_string(), Value::String(new_id.to_string())); + let uuid_str = format!("{}/{}", pli_series_info.url, provider_id); + let uuid = hash_string(uuid_str.as_str()); + let virtual_id = target_id_mapping.insert_entry(provider_id, uuid, &PlaylistItemType::Series, pli_series_info.virtual_id); + episode.insert("id".to_string(), Value::String(virtual_id.to_string())); } } if options.skip_series_direct_source { @@ -309,13 +313,13 @@ fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPl } } if let Ok(result) = serde_json::to_string(&doc) { - let _ = xtream_repository::xtream_write_series_info(config, target.name.replace(' ', "_").as_str(), pli.virtual_id, &new_id_to_provider_id_mapping, &result); + let _ = xtream_repository::xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result); return Ok(result); } } } } - Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli.virtual_id))) + Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli_series_info.virtual_id))) } async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> Result { @@ -325,7 +329,7 @@ async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &ConfigTarget, pli: &XtreamPlaylistItem, info_url: &str, cluster: XtreamCluster) -> Result { if cluster == XtreamCluster::Series { - if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.replace(' ', "_").as_str(), pli.virtual_id) { + if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.as_str(), pli.virtual_id) { return Ok(content); } } @@ -333,12 +337,8 @@ async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &C if let Ok(content) = xtream_get_stream_info_content(info_url, input).await { return match cluster { XtreamCluster::Live => Ok(content), - XtreamCluster::Video => { - return get_xtream_vod_info(target, pli, &content); - } - XtreamCluster::Series => { - return get_xtream_series_info(config, target, pli, &content); - } + XtreamCluster::Video => get_xtream_vod_info(target, pli, &content), + XtreamCluster::Series => get_and_write_xtream_series_info(config, target, pli, &content), }; } @@ -349,12 +349,12 @@ async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &C async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserCredentials, target: &ConfigTarget, stream_id: &str, cluster: XtreamCluster) -> HttpResponse { - let req_stream_id: u32 = match FromStr::from_str(stream_id) { + let virtual_id: u32 = match FromStr::from_str(stream_id) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().finish() }; - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, Some(cluster)) { + if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(cluster)) { let input_id = pli.input_id; if let Some(input) = app_state.config.get_input_by_id(input_id) { if let Some(info_url) = get_xtream_player_api_info_url(input, cluster, pli.provider_id) { @@ -530,9 +530,9 @@ async fn xtream_player_api( } else { let cat_id = if category_id.is_empty() { 0 } else { category_id.parse::().unwrap_or(0) }; match match action { - "get_live_streams" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Live, &app_state.config, target, cat_id), - "get_vod_streams" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Video, &app_state.config, target, cat_id), - "get_series" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Series, &app_state.config, target, cat_id), + "get_live_streams" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Live, &app_state.config, target, cat_id), + "get_vod_streams" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Video, &app_state.config, target, cat_id), + "get_series" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Series, &app_state.config, target, cat_id), _ => Err(Error::new(ErrorKind::Unsupported, format!("Cant find action: {action} for target: {target_name}"))), } { Ok(payload) => HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(payload), diff --git a/src/filter.rs b/src/filter.rs index f488843d4..2502d57bb 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -194,7 +194,7 @@ impl std::fmt::Display for Filter { Filter::TypeComparison(field, item_type) => { write!(f, "{} = {}", field, match item_type { PlaylistItemType::Live => "live", - PlaylistItemType::Movie => "vod", + PlaylistItemType::Video => "vod", PlaylistItemType::Series => "series", PlaylistItemType::SeriesInfo => "series" // yes series-info is handled as series in filter }) @@ -268,8 +268,8 @@ fn get_parser_field_comparison(expr: Pair, templates: &Vec Option { if text_item_type.eq_ignore_ascii_case("live") { Some(PlaylistItemType::Live) - } else if text_item_type.eq_ignore_ascii_case("movie") || text_item_type.eq_ignore_ascii_case("vod") { - Some(PlaylistItemType::Movie) + } else if text_item_type.eq_ignore_ascii_case("movie") || text_item_type.eq_ignore_ascii_case("video") || text_item_type.eq_ignore_ascii_case("vod") { + Some(PlaylistItemType::Video) } else if text_item_type.eq_ignore_ascii_case("series") { Some(PlaylistItemType::Series) } else if text_item_type.eq_ignore_ascii_case("series-info") { diff --git a/src/model/playlist.rs b/src/model/playlist.rs index ec883a60f..0d0e5abe7 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -2,13 +2,15 @@ use std::cell::RefCell; use std::cmp::PartialEq; use std::fmt::{Display, Formatter}; use std::rc::Rc; + use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{ConfigInput, ConfigTarget}; use crate::model::xmltv::TVGuide; use crate::model::xtream::{xtream_playlistitem_to_document, XtreamMappingOptions}; +use crate::processing::m3u_parser::extract_id_from_url; use crate::repository::storage::hash_string; use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32}; @@ -41,6 +43,29 @@ pub(crate) enum XtreamCluster { Series = 3, } +impl XtreamCluster { + // pub fn to_bytes(self) -> [u8; 1] { + // [self as u8] + // } + // + // pub fn from_bytes(bytes: [u8; 1]) -> Result { + // match bytes[0] { + // 1 => Ok(XtreamCluster::Live), + // 2 => Ok(XtreamCluster::Video), + // 3 => Ok(XtreamCluster::Series), + // _ => Err(format!("Invalid value for XtreamCluster: {}", bytes[0])), + // } + // } + + pub fn as_str(&self) -> &str { + match self { + XtreamCluster::Live => "Live", + XtreamCluster::Video => "Video", + XtreamCluster::Series => "Series", + } + } +} + impl Default for XtreamCluster { fn default() -> Self { XtreamCluster::Live @@ -49,22 +74,48 @@ impl Default for XtreamCluster { impl Display for XtreamCluster { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", match self { - XtreamCluster::Live => "live", - XtreamCluster::Video => "movie", - XtreamCluster::Series => "series", - }) + write!(f, "{}", self.as_str()) } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +impl TryFrom for XtreamCluster { + type Error = String; + fn try_from(item_type: PlaylistItemType) -> Result { + match item_type { + PlaylistItemType::Live => Ok(XtreamCluster::Live), + PlaylistItemType::Video => Ok(XtreamCluster::Video), + PlaylistItemType::Series => Ok(XtreamCluster::Series), + PlaylistItemType::SeriesInfo => Err(format!("Cant convert {item_type}")), + } + } +} + +#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[repr(u8)] pub(crate) enum PlaylistItemType { Live = 1, - Movie = 2, + Video = 2, Series = 3, SeriesInfo = 4, } +impl PlaylistItemType { + pub fn to_bytes(self) -> [u8; 1] { + [self as u8] + } + + pub fn from_bytes(bytes: [u8; 1]) -> Result { + match bytes[0] { + 1 => Ok(PlaylistItemType::Live), + 2 => Ok(PlaylistItemType::Video), + 3 => Ok(PlaylistItemType::Series), + 4 => Ok(PlaylistItemType::SeriesInfo), + _ => Err(format!("Invalid value for PlaylistItemType: {}", bytes[0])), + } + } +} + + impl Default for PlaylistItemType { fn default() -> Self { PlaylistItemType::Live @@ -75,7 +126,7 @@ impl From for PlaylistItemType { fn from(xtream_cluster: XtreamCluster) -> Self { match xtream_cluster { XtreamCluster::Live => PlaylistItemType::Live, - XtreamCluster::Video => PlaylistItemType::Movie, + XtreamCluster::Video => PlaylistItemType::Video, XtreamCluster::Series => PlaylistItemType::SeriesInfo, } } @@ -85,7 +136,7 @@ impl Display for PlaylistItemType { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", match self { PlaylistItemType::Live => "live", - PlaylistItemType::Movie => "movie", + PlaylistItemType::Video => "video", PlaylistItemType::Series => "series", PlaylistItemType::SeriesInfo => "series-info", }) @@ -99,7 +150,7 @@ pub(crate) trait FieldAccessor { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct PlaylistItemHeader { - pub uuid: Rc<[u8;32]>, // calculated + pub uuid: Rc<[u8; 32]>, // calculated pub id: Rc, // provider id pub virtual_id: u32, // virtual id pub name: Rc, @@ -127,13 +178,28 @@ pub(crate) struct PlaylistItemHeader { } impl PlaylistItemHeader { - pub(crate) fn gen_uuid(&mut self) { self.uuid = Rc::new(hash_string(&self.url)) } pub(crate) fn get_uuid(&self) -> &Rc<[u8; 32]> { &self.uuid } + + pub(crate) fn get_provider_id(&mut self) -> Option { + match self.id.parse::() { + Ok(id) => Some(id), + Err(_) => match extract_id_from_url(&self.url) { + Some(id) => match id.parse::() { + Ok(newid) => { + self.id = Rc::new(newid.to_string()); + Some(newid) + } + Err(_) => None, + }, + None => None, + } + } + } } macro_rules! to_m3u_non_empty_fields { @@ -325,7 +391,6 @@ pub(crate) struct PlaylistGroup { } impl PlaylistGroup { - pub(crate) fn on_load(&mut self) { self.channels.iter().for_each(|pl| pl.header.borrow_mut().gen_uuid()); } diff --git a/src/processing/m3u_parser.rs b/src/processing/m3u_parser.rs index ba1cb96e0..e366f5d7f 100644 --- a/src/processing/m3u_parser.rs +++ b/src/processing/m3u_parser.rs @@ -136,7 +136,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { // TODO find Series based on group or configured names plih.xtream_cluster = XtreamCluster::Video; - plih.item_type = PlaylistItemType::Movie; + plih.item_type = PlaylistItemType::Video; } { diff --git a/src/repository/index_record.rs b/src/repository/index_record.rs deleted file mode 100644 index c4cf37ac1..000000000 --- a/src/repository/index_record.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::fs::File; -use std::io::{Error, Read, Seek, SeekFrom, Write}; -use std::path::Path; -use crate::utils::file_utils; - -/** -We write the structs with `bincode::encode` to a file. -To access each entry we need a index file where we can find the -Entries with offset and size of the encoded struct. -This is used for the index file where first entry is the index -of the encoded file, and size is the size of the encoded struct. - -We also use it for different purposes, like storing id -> to id mapping. - */ -pub(in crate::repository) struct IndexRecord { - pub left: u32, - pub right: u32, -} - -impl IndexRecord { - pub fn from_file(file: &mut File, offset: u32) -> Result { - file.seek(SeekFrom::Start(u64::from(offset)))?; - let mut left_bytes = [0u8; 4]; - let mut right_bytes = [0u8; 4]; - file.read_exact(&mut left_bytes)?; - file.read_exact(&mut right_bytes)?; - let left = u32::from_le_bytes(left_bytes); - let right = u32::from_le_bytes(right_bytes); - Ok(IndexRecord { left, right }) - } - - pub fn to_file(path: &Path, left: u32, right: u32, append: bool) -> Result<(), Error> { - match file_utils::open_file_append(path, append) { - Ok(mut file) => { - let bytes = IndexRecord::to_bytes(left, right); - file.write_all(&bytes) - }, - Err(err) => Err(err) - } - } - - pub fn to_bytes(left: u32, right: u32) -> [u8; 8] { - let left_bytes: [u8; 4] = left.to_le_bytes(); - let right_bytes: [u8; 4] = right.to_le_bytes(); - let mut combined_bytes: [u8; 8] = [0; 8]; - combined_bytes[..4].copy_from_slice(&left_bytes); - combined_bytes[4..].copy_from_slice(&right_bytes); - combined_bytes - } - - pub fn get_record_size() -> u32 { 8 } - pub fn get_index_offset(index: u32) -> u32 { index * 8 } -} \ No newline at end of file diff --git a/src/repository/indexed_document_reader.rs b/src/repository/indexed_document_reader.rs index ee727f5d7..702786bc1 100644 --- a/src/repository/indexed_document_reader.rs +++ b/src/repository/indexed_document_reader.rs @@ -3,12 +3,18 @@ use std::fs::File; use std::io::{Error, ErrorKind, Read, Seek, SeekFrom}; use std::marker::PhantomData; use std::path::Path; +use crate::repository::indexed_document_writer::get_record_size; -use crate::repository::index_record::IndexRecord; + +fn read_content_size(main_file: &mut File) -> Result { + let mut size_bytes = [0u8; 4]; + main_file.read_exact(&mut size_bytes)?; + let buf_size = u32::from_le_bytes(size_bytes) as usize; + Ok(buf_size) +} pub(in crate::repository) struct IndexedDocumentReader { main_file: File, - index_file: File, cursor: u32, size: u32, failed: bool, @@ -31,7 +37,6 @@ impl IndexedDocumentReader { }; Ok(Self { main_file, - index_file, cursor: 0, size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?, failed: false, @@ -59,33 +64,31 @@ impl IndexedDocumentReader { !self.failed && self.cursor < self.size } pub fn read_next(&mut self) -> Result, Error> { - if self.has_next() { - let record = IndexRecord::from_file(&mut self.index_file, self.cursor); - self.cursor += IndexRecord::get_record_size(); - match record { - Ok(index_record) => { - let offset = u64::from(index_record.left); - let buf_size = index_record.right as usize; - if self.t_buffer.len() < buf_size { - self.t_buffer.resize(buf_size, 0u8); - } - self.main_file.seek(SeekFrom::Start(offset))?; - self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?; - return match bincode::deserialize::(&self.t_buffer[0..buf_size]) { - Ok(value) => Ok(Some(value)), - Err(err) => { - self.failed = true; - Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))) - } - }; - } - Err(err) => { - self.failed = true; - return Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))); - } - } + if !self.has_next() { + return Ok(None); } - Ok(None) + // read content-size + let buf_size: usize = read_content_size(&mut self.main_file)?; + // resize buffer if necessary + if self.t_buffer.capacity() < buf_size { + self.t_buffer.reserve(buf_size - self.t_buffer.capacity()); + } + self.t_buffer.resize(buf_size, 0u8); + + // read content + self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?; + + // deserialize buffer + return match bincode::deserialize::(&self.t_buffer[0..buf_size]) { + Ok(value) => { + self.cursor += get_record_size(); + Ok(Some(value)) + } + Err(err) => { + self.failed = true; + Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))) + } + }; } } @@ -103,19 +106,34 @@ impl Iterator for IndexedDocumentReader } } -pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, offset: u32) -> Result - where T: ?Sized + serde::de::DeserializeOwned +fn get_offset(file: &mut File, doc_id: u32) -> Result { + // TODO optimize this, sequential searching trough complete file! + let mut bytes = [0u8; 8]; + file.seek(SeekFrom::Start(0))?; + loop { + file.read_exact(&mut bytes)?; + let id = u32::from_le_bytes(bytes[..4].try_into().unwrap()); + if id == doc_id { + return Ok(u64::from(u32::from_le_bytes(bytes[4..].try_into().unwrap()))); + } + } +} + +pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, doc_id: u32) -> Result +where + T: ?Sized + serde::de::DeserializeOwned, { if main_path.exists() && index_path.exists() { let mut index_file = File::open(index_path)?; let mut main_file = File::open(main_path)?; - let index_record = IndexRecord::from_file(&mut index_file, offset)?; - main_file.seek(SeekFrom::Start(u64::from(index_record.left)))?; - let mut buffer: Vec = vec![0; index_record.right as usize]; + let offset = get_offset(&mut index_file, doc_id)?; + main_file.seek(SeekFrom::Start(offset))?; + let buf_size = read_content_size(&mut main_file)?; + let mut buffer: Vec = vec![0; buf_size]; main_file.read_exact(&mut buffer)?; if let Ok(item) = bincode::deserialize::(&buffer[..]) { return Ok(item); } } - Err(Error::new(ErrorKind::Other, format!("Failed to read item for offset {} - {}", offset, main_path.to_str().unwrap()))) -} \ No newline at end of file + Err(Error::new(ErrorKind::Other, format!("Failed to read item for id {} - {}", doc_id, main_path.to_str().unwrap()))) +} diff --git a/src/repository/indexed_document_writer.rs b/src/repository/indexed_document_writer.rs index b05d7439f..2c55d6c2d 100644 --- a/src/repository/indexed_document_writer.rs +++ b/src/repository/indexed_document_writer.rs @@ -3,10 +3,31 @@ use std::fs::File; use std::io::{Error, ErrorKind, Write}; use std::path::PathBuf; -use crate::repository::index_record::IndexRecord; use crate::utils::file_utils; use crate::utils::file_utils::create_file_tuple; +pub(in crate::repository) fn get_record_size() -> u32 { 8 } + +fn to_bytes(left: u32, right: u32) -> [u8; 8] { + let left_bytes: [u8; 4] = left.to_le_bytes(); + let right_bytes: [u8; 4] = right.to_le_bytes(); + let mut combined_bytes: [u8; 8] = [0; 8]; + combined_bytes[..4].copy_from_slice(&left_bytes); + combined_bytes[4..].copy_from_slice(&right_bytes); + combined_bytes +} + +/** +* Creates two files, +* - content +* - index +* +* Layout of content file record is: +* - content-size (u32) + content +* +* Layout of index file record is: +* - document_id (u32) + content file offset + (u32) +*/ pub(in crate::repository) struct IndexedDocumentWriter { main_path: PathBuf, index_path: PathBuf, @@ -55,15 +76,18 @@ impl IndexedDocumentWriter { let current_main_index = self.main_offset; let current_index_index = self.index_offset; if let Ok(encoded) = bincode::serialize(doc) { - match file_utils::check_write(&self.main_file.write_all(&encoded)) { + let content_bytes = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; + let mut data: Vec = content_bytes.to_le_bytes().to_vec(); + data.extend(&encoded); + match file_utils::check_write(&self.main_file.write_all(&data)) { Ok(()) => { - let bytes_written = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; - let combined_bytes = IndexRecord::to_bytes(doc_id, bytes_written); + let combined_bytes = to_bytes(doc_id, self.main_offset); if let Err(err) = file_utils::check_write(&self.index_file.write_all(&combined_bytes)) { return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.index_path.to_str().unwrap(), err))); } - self.main_offset += bytes_written; - self.index_offset += IndexRecord::get_record_size(); + let written_bytes = u32::try_from(data.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; + self.main_offset += written_bytes; + self.index_offset += get_record_size(); } Err(err) => { return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.main_path.to_str().unwrap(), err))); diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 214c73b4d..26fa249d7 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -10,7 +10,6 @@ use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{M3uPlaylistItem, PlaylistGroup, PlaylistItem, PlaylistItemType}; -use crate::repository::index_record::IndexRecord; use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; use crate::repository::indexed_document_writer::IndexedDocumentWriter; use crate::repository::storage::{ensure_target_storage_path}; @@ -135,5 +134,5 @@ pub(crate) fn m3u_get_item_for_stream_id(stream_id: u32, m3u_path: &Path, idx_pa if stream_id < 1 { return Err(Error::new(ErrorKind::Other, "id should start with 1")); } - read_indexed_item::(m3u_path, idx_path, IndexRecord::get_index_offset(stream_id - 1)) + read_indexed_item::(m3u_path, idx_path, stream_id) } \ No newline at end of file diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 8ca12a2fa..8ffa7c0d8 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -5,7 +5,6 @@ pub(crate) mod epg_repository; pub(crate) mod kodi_repository; pub(crate) mod storage; -mod index_record; mod indexed_document_writer; mod indexed_document_reader; -mod target_id_mapping_record; \ No newline at end of file +pub(crate) mod target_id_mapping_record; \ No newline at end of file diff --git a/src/repository/playlist_repository.rs b/src/repository/playlist_repository.rs index 51662fc53..e7ded5468 100644 --- a/src/repository/playlist_repository.rs +++ b/src/repository/playlist_repository.rs @@ -20,13 +20,14 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> for group in &mut *playlist { for channel in &group.channels { let mut header = channel.header.borrow_mut(); - match header.id.parse::() { - Ok(provider_id) => { + match header.get_provider_id() { + Some(provider_id) => { let uuid = header.get_uuid(); - header.virtual_id = target_id_mapping.insert_entry(provider_id, **uuid); + let item_type = header.item_type; + header.virtual_id = target_id_mapping.insert_entry(provider_id, **uuid, &item_type, 0); } - Err(err) => { - errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + None => { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Playlistitem has no provider id: {}", &header.title))); } } } @@ -35,7 +36,7 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> for output in &target.output { match match output.target { TargetType::M3u => m3u_write_playlist(target, cfg, &target_path, playlist), - TargetType::Xtream => xtream_write_playlist(target, cfg, &target_path, playlist), + TargetType::Xtream => xtream_write_playlist(target, cfg, playlist), TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), } { Ok(()) => { @@ -52,9 +53,14 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> Err(err) => errors.push(err) } } + if let Err(err) = target_id_mapping.persist() { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } } - Err(err) => errors.push(err), - } + Err(err) => { + errors.push(err); + }, + }; if errors.is_empty() { Ok(()) } else { Err(errors) } } diff --git a/src/repository/target_id_mapping_record.rs b/src/repository/target_id_mapping_record.rs index 7e638ddae..565fdec2f 100644 --- a/src/repository/target_id_mapping_record.rs +++ b/src/repository/target_id_mapping_record.rs @@ -1,9 +1,11 @@ use std::cmp::max; use std::collections::BTreeMap; use std::fs::File; -use std::io::{Error, Read, Seek, SeekFrom, Write}; +use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write}; use std::path::Path; use std::rc::Rc; +use actix_web::body::MessageBody; +use crate::model::playlist::{PlaylistItemType}; use crate::utils::file_utils; @@ -11,42 +13,67 @@ use crate::utils::file_utils; This file contains the provider id, the virtual id, and an uuid */ -pub(in crate::repository) struct TargetIdMappingRecord { +pub(crate) struct TargetIdMappingRecord { pub provider_id: u32, pub virtual_id: u32, pub uuid: [u8; 32], + pub item_type: PlaylistItemType, + pub parent_virtual_id: u32, // only for series to hold series info id. } impl TargetIdMappingRecord { - fn to_bytes(&self) -> [u8; 40] { + fn to_bytes(&self) -> [u8; 45] { let provider_id_bytes: [u8; 4] = self.provider_id.to_le_bytes(); let virtual_id_bytes: [u8; 4] = self.virtual_id.to_le_bytes(); - let mut combined_bytes: [u8; 40] = [0; 40]; - combined_bytes[..4].copy_from_slice(&provider_id_bytes); - combined_bytes[4..].copy_from_slice(&virtual_id_bytes); - combined_bytes[8..].copy_from_slice(&self.uuid); + let parent_virtual_id_bytes: [u8; 4] = self.parent_virtual_id.to_le_bytes(); + let item_type_bytes: [u8; 1] = self.item_type.to_bytes(); + + let mut combined_bytes: [u8; 45] = [0; 45]; + combined_bytes[0..4].copy_from_slice(&provider_id_bytes); + combined_bytes[4..8].copy_from_slice(&virtual_id_bytes); + combined_bytes[8..12].copy_from_slice(&parent_virtual_id_bytes); + combined_bytes[12..13].copy_from_slice(&item_type_bytes); + combined_bytes[13..45].copy_from_slice(&self.uuid); + combined_bytes } + + fn from_bytes(bytes: &[u8; 45]) -> Self { + let provider_id_bytes: [u8; 4] = bytes[0..4].try_into().expect("Slice with incorrect length"); + let virtual_id_bytes: [u8; 4] = bytes[4..8].try_into().expect("Slice with incorrect length"); + let parent_virtual_id_bytes: [u8; 4] = bytes[8..12].try_into().expect("Slice with incorrect length"); + let item_type_bytes: [u8; 1] = bytes[12..13].try_into().expect("Slice with incorrect length"); + let uuid: [u8; 32] = bytes[13..45].try_into().expect("Slice with incorrect length"); + + TargetIdMappingRecord { + provider_id: u32::from_le_bytes(provider_id_bytes), + virtual_id: u32::from_le_bytes(virtual_id_bytes), + parent_virtual_id: u32::from_le_bytes(parent_virtual_id_bytes), + item_type: PlaylistItemType::from_bytes(item_type_bytes).unwrap(), + uuid, + } + } } -pub(in crate::repository) struct TargetIdMapping { +pub(crate) struct TargetIdMapping { dirty: bool, virtual_id_counter: u32, by_provider_id: BTreeMap>, by_virtual_id: BTreeMap>, by_uuid: BTreeMap<[u8; 32], Rc>, records: Vec>, + file: Option } impl TargetIdMapping { - pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32]) -> u32 { + pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32], item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { self.dirty = true; self.virtual_id_counter += 1; - self.insert(TargetIdMappingRecord { provider_id, virtual_id: self.virtual_id_counter, uuid }); + self.insert(TargetIdMappingRecord { provider_id, virtual_id: self.virtual_id_counter, uuid, item_type: *item_type, parent_virtual_id}); self.virtual_id_counter } - fn new(records: Vec>) -> Self { + fn new(records: Vec>, file: Option) -> Self { let mut by_provider_id: BTreeMap> = BTreeMap::new(); let mut by_virtual_id: BTreeMap> = BTreeMap::new(); let mut by_uuid: BTreeMap<[u8; 32], Rc> = BTreeMap::new(); @@ -64,6 +91,7 @@ impl TargetIdMapping { by_virtual_id, by_uuid, records, + file } } @@ -78,36 +106,42 @@ impl TargetIdMapping { self.records.push(shared_record); } - fn get_by_provider_id(&self, provider_id: u32) -> Option<&Rc> { + pub(crate) fn get_by_provider_id(&self, provider_id: u32) -> Option<&Rc> { self.by_provider_id.get(&provider_id) } - fn get_by_virtual_id(&self, virtual_id: u32) -> Option<&Rc> { + pub(crate) fn get_by_virtual_id(&self, virtual_id: u32) -> Option<&Rc> { self.by_virtual_id.get(&virtual_id) } - fn get_by_uuid(&self, uuid: &[u8; 32]) -> Option<&Rc> { + pub(crate) fn get_by_uuid(&self, uuid: &[u8; 32]) -> Option<&Rc> { self.by_uuid.get(uuid) } + + pub fn persist(&mut self) -> Result<(), Error> { + let mut file = match self.file.take() { + Some(file) => file, + None => return Err(Error::new(ErrorKind::NotFound, "No file given")), + }; + let result = self.to_file(&mut file); + self.file = Some(file); + result + } + pub fn to_file(&mut self, file: &mut File) -> Result<(), Error> { - for record in &self.records { - let bytes = record.to_bytes(); - if let Err(err) = file.write_all(&bytes) { - return Err(err); + if self.dirty { + for record in &self.records { + let bytes = record.to_bytes(); + if let Err(err) = file.write_all(&bytes) { + return Err(err); + } } } self.dirty = false; Ok(()) } - pub fn from_path(path: &Path) -> Self { - match file_utils::open_file_append(path, false) { - Ok(mut file) => TargetIdMapping::from_file(&mut file), - _ => TargetIdMapping::new(vec![]) - } - } - pub fn to_path(&mut self, path: &Path) -> Result<(), Error> { match file_utils::open_file_append(path, false) { Ok(mut file) => self.to_file(&mut file), @@ -115,27 +149,27 @@ impl TargetIdMapping { } } - pub fn from_file(file: &mut File) -> Self { + pub fn from_path(path: &Path) -> Self { + match file_utils::open_file_append(path, false) { + Ok(mut file) => { + let records = TargetIdMapping::read_records_from_file(&mut file); + TargetIdMapping::new(records, Some(file)) + }, + _ => TargetIdMapping::new(vec![], None) + } + } + + fn read_records_from_file(file: &mut File) -> Vec> { let mut records = vec![]; if let Ok(_) = file.seek(SeekFrom::Start(0)) { + let mut bytes = [0u8;45]; loop { - let mut provider_id_bytes = [0u8; 4]; - let mut virtual_id_bytes = [0u8; 4]; - let mut uuid = [0u8; 32]; - if let Err(_) = file.read_exact(&mut provider_id_bytes) { + if let Err(_) = file.read_exact(&mut bytes) { break; } - if let Err(_) = file.read_exact(&mut virtual_id_bytes) { - break; - } - if let Err(_) = file.read_exact(&mut uuid) { - break; - } - let provider_id = u32::from_le_bytes(provider_id_bytes); - let virtual_id = u32::from_le_bytes(virtual_id_bytes); - records.push(Rc::new(TargetIdMappingRecord { provider_id, virtual_id, uuid })); + records.push(Rc::new(TargetIdMappingRecord::from_bytes(&bytes))); } } - TargetIdMapping::new(records) + records } } \ No newline at end of file diff --git a/src/repository/xtream_repository.rs b/src/repository/xtream_repository.rs index 7fbed66c8..70d9152d9 100644 --- a/src/repository/xtream_repository.rs +++ b/src/repository/xtream_repository.rs @@ -1,9 +1,8 @@ use std::collections::HashMap; use std::convert::TryInto; use std::fs::File; -use std::io::{BufReader, Error, ErrorKind, Read, Seek, SeekFrom, Write}; +use std::io::{BufReader, Error, ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; -use std::rc::Rc; use log::error; use serde_json::{json, Value}; @@ -13,20 +12,16 @@ use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; -use crate::processing::m3u_parser::extract_id_from_url; -use crate::repository::index_record::IndexRecord; use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; use crate::repository::indexed_document_writer::IndexedDocumentWriter; -use crate::repository::storage::get_target_storage_path; +use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path}; +use crate::repository::target_id_mapping_record::TargetIdMapping; use crate::utils::file_utils; use crate::utils::json_utils::{json_iter_array, json_write_documents_to_file}; pub(crate) static COL_CAT_LIVE: &str = "cat_live"; pub(crate) static COL_CAT_SERIES: &str = "cat_series"; pub(crate) static COL_CAT_VOD: &str = "cat_vod"; -pub(crate) static COL_LIVE: &str = "live"; -pub(crate) static COL_SERIES: &str = "series"; -pub(crate) static COL_VOD: &str = "vod"; macro_rules! cant_write_result { ($path:expr, $err:expr) => { @@ -38,10 +33,6 @@ fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } -pub(crate) fn xtream_get_stream_id_cluster_index_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("cluster_index.db") -} - // Maps episode id -> to provider_episode_id and series_id fn xtream_get_series_episode_id_mapping_file_path(storage_path: &Path) -> PathBuf { storage_path.join("mapping_episode.db") @@ -95,67 +86,34 @@ fn xtream_clear_catchup(storage_path: &Path) { let _ = std::fs::remove_file(xtream_get_catchup_id_mapping_file_path(storage_path)); } -fn write_playlist_to_file(storage_path: &Path, stream_id: &mut u32, cluster: XtreamCluster, playlist: &mut [PlaylistItem]) -> Result<(), M3uFilterError> { - let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); - match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { - Ok(mut writer) => { - for pli in playlist.iter_mut() { - if let Ok(mut xtream) = pli.to_xtream() { - xtream.virtual_id = *stream_id; - match writer.write_doc(xtream.virtual_id, &xtream) { - Ok(_) => *stream_id += 1, +fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { + for (cluster, playlist) in collections { + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, &cluster); + match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { + Ok(mut writer) => { + for item in playlist { + match item.to_xtream() { + Ok(xtream) => { + match writer.write_doc(item.header.borrow().virtual_id, &xtream) { + Ok(_) => {} + Err(err) => return cant_write_result!(&xtream_path, err) + } + }, Err(err) => return cant_write_result!(&xtream_path, err) } } } - if cluster == XtreamCluster::Live { - xtream_clear_catchup(storage_path); - } else if cluster == XtreamCluster::Series { - xtream_clear_series_info(storage_path); - } - Ok(()) + Err(err) => return cant_write_result!(&xtream_path, err) } - Err(err) => cant_write_result!(&xtream_path, err) - } -} - -fn save_stream_id_cluster_mapping(storage_path: &Path, id_data: &mut Vec<(XtreamCluster, u32, u32)>) -> Result<(), Error> { - let stream_id_path = xtream_get_stream_id_cluster_index_file_path(storage_path); - id_data.sort_by(|(_, start_a, _), (_, start_b, _)| start_b.cmp(start_a)); - let encoded: Vec = bincode::serialize(id_data).unwrap(); - std::fs::write(stream_id_path, encoded) -} - -fn load_stream_id_cluster_mapping(storage_path: &Path) -> Option> { - let path = xtream_get_stream_id_cluster_index_file_path(storage_path); - if path.exists() { - match std::fs::read(path) { - Ok(encoded) => { - let decoded: Vec<(XtreamCluster, u32, u32)> = bincode::deserialize(&encoded[..]).unwrap(); - Some(decoded) - } - Err(_) => None, + if cluster == XtreamCluster::Live { + xtream_clear_catchup(storage_path); + } else if cluster == XtreamCluster::Series { + xtream_clear_series_info(storage_path); } - } else { - None } + Ok(()) } -fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { - let mut id_list: Vec<(XtreamCluster, u32, u32)> = vec![]; - let mut stream_id: u32 = 1; - for (cluster, playlist) in collections { - let start = stream_id; - write_playlist_to_file(storage_path, &mut stream_id, cluster, playlist)?; - id_list.push((cluster, start, stream_id)); - } - match save_stream_id_cluster_mapping(storage_path, &mut id_list) { - Ok(()) => Ok(()), - Err(err) => Err(M3uFilterError::new(M3uFilterErrorKind::Notify, format!("failed to write xtream playlist: {} - {}", storage_path.to_str().unwrap(), err))) - } -} - - fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Option { if let Some(value) = map.get(key) { if let Some(result) = value.as_str() { @@ -204,19 +162,23 @@ pub(crate) fn xtream_get_epg_file_path(path: &Path) -> PathBuf { path.join("epg.xml") } -pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: XtreamCluster) -> (PathBuf, PathBuf) { - let xtream_path = storage_path.join(format!("{}.db", match cluster { - XtreamCluster::Live => COL_LIVE, - XtreamCluster::Video => COL_VOD, - XtreamCluster::Series => COL_SERIES - })); +fn xtream_get_file_paths_for_name(storage_path: &Path, name: &str) -> (PathBuf, PathBuf) { + let xtream_path = storage_path.join(format!("{}.db", name)); let extension = xtream_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or(""))); let index_path = xtream_path.with_extension(format!("{}idx", &extension.unwrap_or_default())); (xtream_path, index_path) } -pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, target_path: &Path, playlist: &mut [PlaylistGroup]) -> Result<(), M3uFilterError> { - match ensure_xtream_storage_path(cfg, target.name.replace(' ', "_").as_str()) { +pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: &XtreamCluster) -> (PathBuf, PathBuf) { + xtream_get_file_paths_for_name(storage_path, &cluster.as_str().to_lowercase()) +} + +pub(crate) fn xtream_get_file_paths_for_series(storage_path: &Path) -> (PathBuf, PathBuf) { + xtream_get_file_paths_for_name(storage_path, "series") +} + +pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut [PlaylistGroup]) -> Result<(), M3uFilterError> { + match ensure_xtream_storage_path(cfg, target.name.as_str()) { Ok(path) => { let mut cat_live_col = vec![]; let mut cat_series_col = vec![]; @@ -225,7 +187,7 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, target_ let mut series_col = vec![]; let mut vod_col = vec![]; let mut errors = Vec::new(); -// TODO.EUZU old catgeory ids + // preserve category_ids let (max_cat_id, existing_cat_ids) = load_old_category_ids(&path); let mut cat_id_counter = max_cat_id; @@ -253,30 +215,8 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, target_ let col = if header.item_type == PlaylistItemType::Series { None } else { - if header.id.parse::().is_err() { - let id_from_url = match extract_id_from_url(&header.url) { - Some(id) => match id.parse::() { - Ok(newid) => Some(newid), - Err(_) => None, - }, - None => None, - }; - - let has_id = match id_from_url { - Some(newid) => { - header.id = Rc::new(newid.to_string()); - Ok(()) - } - None => { - let title = header.title.as_str(); - errors.push(format!("Channel does not have an id: {title}")); - Err(()) - } - }; - // Instead of returning from the function, handle the error by assigning None to col. - if has_id.is_err() { - None - } else { + match header.get_provider_id() { + Some(_) => { header.category_id = *cat_id; Some(match header.xtream_cluster { XtreamCluster::Live => &mut live_col, @@ -284,13 +224,11 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, target_ XtreamCluster::Video => &mut vod_col, }) } - } else { - header.category_id = *cat_id; - Some(match header.xtream_cluster { - XtreamCluster::Live => &mut live_col, - XtreamCluster::Series => &mut series_col, - XtreamCluster::Video => &mut vod_col, - }) + None => { + let title = header.title.as_str(); + errors.push(format!("Channel does not have an id: {title}")); + None + } } }; drop(header); @@ -342,44 +280,45 @@ pub(crate) fn xtream_get_collection_path(cfg: &Config, target_name: &str, collec Err(Error::new(ErrorKind::Other, format!("Cant find collection: {target_name}/{collection_name}"))) } -fn _xtream_get_item_for_stream_id(stream_id: u32, storage_path: &Path, xtream_cluster: Option, mapping: &[(XtreamCluster, u32, u32)]) -> Result { - if let Some((cluster, cluster_start, _end)) = match xtream_cluster { - Some(clus) => mapping.iter().find(|(c, _, _)| *c == clus), - None => mapping.iter().find(|(_cluster, start, _end)| stream_id >= *start), - } { - let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, *cluster); - if stream_id >= *cluster_start { - return read_indexed_item::(&xtream_path, &idx_path, IndexRecord::get_index_offset(stream_id - cluster_start)); - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}"))) +fn xtream_read_item_for_stream_id(stream_id: u32, storage_path: &Path, cluster: &XtreamCluster) -> Result { + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); + return read_indexed_item::(&xtream_path, &idx_path, stream_id); +} + +fn xtream_read_series_item_for_stream_id(stream_id: u32, storage_path: &Path) -> Result { + let (xtream_path, idx_path) = xtream_get_file_paths_for_series(storage_path); + return read_indexed_item::(&xtream_path, &idx_path, stream_id); } pub(crate) fn xtream_get_item_for_stream_id(stream_id: u32, config: &Config, target: &ConfigTarget, xtream_cluster: Option) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target.name.replace(' ', "_").as_str()) { - if let Some(mapping) = load_stream_id_cluster_mapping(&storage_path) { - if let Some(max) = mapping.iter().map(|(_cluster, _start, end)| end).max().copied() { - if max < stream_id { - // episoden id's fangen bei (max cluster id + 1) an. - // episode mapping har 3 u32 also 12 bytes - let index = stream_id - (max + 1); - if let Ok((_episode_id, provider_id, series_id)) = xtream_read_episode_id_mapping(&storage_path, index) { - if let Ok(mut pli) = _xtream_get_item_for_stream_id(series_id, &storage_path, xtream_cluster, &mapping) { - pli.provider_id = provider_id; - return Ok(pli); + if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { + let cluster = match xtream_cluster { + Some(c) => c, + None => { + if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { + let target_id_mapping = TargetIdMapping::from_path(&get_target_id_mapping_file(&target_path)); + if let Some(mapping) = target_id_mapping.get_by_virtual_id(stream_id) { + if mapping.item_type == PlaylistItemType::SeriesInfo { + return xtream_read_series_item_for_stream_id(stream_id, &storage_path); + } else { + XtreamCluster::try_from(mapping.item_type).unwrap() } + } else { + return Err(Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {stream_id}"))); } - return Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}"))); + } else { + return Err(Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name))); } } - return _xtream_get_item_for_stream_id(stream_id, &storage_path, xtream_cluster, &mapping); - } + }; + xtream_read_item_for_stream_id(stream_id, &storage_path, &cluster) + } else { + Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}, storage path not found"))) } - Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}"))) } -pub(crate) fn xtream_load_rewrite_playlist(cluster: XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target.name.replace(' ', "_").as_str()) { +pub(crate) fn xtream_load_rewrite_playlist(cluster: &XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result { + if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { let (xtream_path, idx_path) = xtream_get_file_paths(&storage_path, cluster); match IndexedDocumentReader::::new(&xtream_path, &idx_path) { Ok(mut reader) => { @@ -400,91 +339,17 @@ pub(crate) fn xtream_load_rewrite_playlist(cluster: XtreamCluster, config: &Conf Err(Error::new(ErrorKind::Other, format!("Failed to find xtream storage for target {}", &target.name))) } -// The mapping file record is episode_id 4bytes, episode_provider_id 4bytes, series_id 4bytes -fn read_last_id_from_episode_id_mapping_file(path: &Path) -> Result { - let mut file = File::open(path)?; - file.seek(SeekFrom::End(-12))?; - let mut buffer = [0; 4]; - file.read_exact(&mut buffer)?; - Ok(u32::from_le_bytes(buffer)) -} - -pub(crate) fn xtream_get_max_series_info_episode_id(config: &Config, target_name: &str) -> Option { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let mapping_file_path = xtream_get_series_episode_id_mapping_file_path(&storage_path); - if mapping_file_path.exists() { - if let Ok(last_id) = read_last_id_from_episode_id_mapping_file(&mapping_file_path) { - return Some(last_id); - } - } - - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some(mapping) = load_stream_id_cluster_mapping(&storage_path) { - return mapping.iter().map(|(_, _, end)| end).max().copied(); - } - } - } - None -} - -fn xtream_write_episode_id_mapping(storage_path: &Path, series_id: u32, episode_id_mapping: &[(u32, u32)]) -> Result<(), Error> { - let file_path = xtream_get_series_episode_id_mapping_file_path(storage_path); - if let Ok(mut file) = if file_path.exists() { - std::fs::OpenOptions::new() - .append(true) // Open in append mode - .open(file_path) - } else { - File::create(file_path) - } { - let stream_id_bytes: [u8; 4] = series_id.to_le_bytes(); - let mut bytes: Vec = Vec::new(); - for (episode_id, provider_id) in episode_id_mapping { - bytes.extend_from_slice(&episode_id.to_le_bytes()); - bytes.extend_from_slice(&provider_id.to_le_bytes()); - bytes.extend_from_slice(&stream_id_bytes); - } - return file_utils::check_write(&file.write_all(&bytes[..])); - } - Err(Error::new(ErrorKind::Other, format!("Failed to open series info idmapping file inside {}", storage_path.to_str().unwrap()))) -} - -fn xtream_read_episode_id_mapping(storage_path: &Path, index: u32) -> Result<(u32, u32, u32), Error> { - let offset = index * 12; - let episode_file = xtream_get_series_episode_id_mapping_file_path(storage_path); - if episode_file.exists() { - if let Ok(mut file) = File::open(episode_file) { - file.seek(SeekFrom::Start(u64::from(offset)))?; - let mut bytes = [0u8; 4]; - file.read_exact(&mut bytes)?; - let episode_id = u32::from_le_bytes(bytes); - file.read_exact(&mut bytes)?; - let provider_id = u32::from_le_bytes(bytes); - file.read_exact(&mut bytes)?; - let series_id = u32::from_le_bytes(bytes); - Ok((episode_id, provider_id, series_id)) - } else { - Err(Error::new(ErrorKind::Other, format!("Could not find episode mapping at offset {offset}"))) - } - } else { - Err(Error::new(ErrorKind::Other, format!("Episode mapping not found at {}", storage_path.to_str().unwrap_or("")))) - } -} - pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, - series_id: u32, episode_id_mapping: &[(u32, u32)], + series_id: u32, content: &str) -> Result<(), Error> { if let Some(storage_path) = xtream_get_storage_path(config, target_name) { if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { return match IndexedDocumentWriter::new_append(info_path.clone(), idx_path) { Ok(mut writer) => { match writer.write_doc(series_id, content) { - Ok((_, index_offset)) => { - let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path); - IndexRecord::to_file(&series_id_index_mapping_path, series_id, index_offset, true)?; - } + Ok(_) => Ok(()), Err(_) => return Err(Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}"))) } - xtream_write_episode_id_mapping(&storage_path, series_id, episode_id_mapping) } Err(err) => Err(err) }; @@ -493,6 +358,7 @@ pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, Ok(()) } +// Reads the series info entry if exists, otherwise error pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Result { if let Some(storage_path) = xtream_get_storage_path(config, target_name) { let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path); diff --git a/test/rest-api.http b/test/rest-api.http index 01c45fe29..cb51d05e4 100644 --- a/test/rest-api.http +++ b/test/rest-api.http @@ -17,10 +17,10 @@ Content-Type: application/json {"username": "test", "password": "test"} ### xtream live_categories -GET http://localhost:8901/m3u?username=me&password=me +GET http://localhost:8901/player_api.php?action=get_live_categories&username=xt&password=xt ### xtream live_streams -GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&password=xt&category_id=203 +GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&password=xt&category_id=102 ### xtream series_categories GET http://localhost:8901/player_api.php?action=get_series_categories&username=xt&password=xt From 8fa2669c0a6c1f2164fa898ef50a4345f04f4b3d Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 16:12:50 +0100 Subject: [PATCH 03/11] added new BPlusTree implementation with store data as blocks implementation --- src/repository/target_id_mapping_record.rs | 1 - src/test.rs | 3 + src/utils/bplustree.rs | 378 +++++++++++++++++++++ src/utils/mod.rs | 1 + 4 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 src/utils/bplustree.rs diff --git a/src/repository/target_id_mapping_record.rs b/src/repository/target_id_mapping_record.rs index 565fdec2f..803a04203 100644 --- a/src/repository/target_id_mapping_record.rs +++ b/src/repository/target_id_mapping_record.rs @@ -4,7 +4,6 @@ use std::fs::File; use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write}; use std::path::Path; use std::rc::Rc; -use actix_web::body::MessageBody; use crate::model::playlist::{PlaylistItemType}; use crate::utils::file_utils; diff --git a/src/test.rs b/src/test.rs index cd7de9006..5c71cdd15 100644 --- a/src/test.rs +++ b/src/test.rs @@ -1,6 +1,7 @@ #[cfg(test)] mod tests { use std::cell::RefCell; + use std::rc::Rc; use regex::Regex; use crate::filter::{get_filter, MockValueProcessor, ValueProvider}; use crate::model::playlist::{PlaylistItem, PlaylistItemHeader}; @@ -8,6 +9,8 @@ mod tests { fn create_mock_pli(name: &str, group: &str) -> PlaylistItem { PlaylistItem { header: RefCell::new(PlaylistItemHeader { + name: Rc::new(name.to_string()), + group: Rc::new(group.to_string()), ..Default::default() }) } diff --git a/src/utils/bplustree.rs b/src/utils/bplustree.rs new file mode 100644 index 000000000..6f6d86ad2 --- /dev/null +++ b/src/utils/bplustree.rs @@ -0,0 +1,378 @@ +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; + +use serde::{Deserialize, Serialize}; + +const BINCODE_OVERHEAD: usize = 4; +const BLOCK_SIZE: usize = 4096; +const POINTER_SIZE: usize = size_of::>(); +// +// #[derive(Serialize, Deserialize, Debug, Clone)] +// struct LeafKeyValue { +// key: K, +// value: V, +// } +// +// #[derive(Serialize, Deserialize, Debug, Clone)] +// struct InnerKeyValue { +// key: K, +// value: V, +// children: u64 +// } +// + +#[derive(Serialize, Deserialize, Debug, Clone)] +struct BPlusTreeNode { + keys: Vec, + children: Vec>, + is_leaf: bool, + values: Vec, // only used in leaf nodes +} + +impl BPlusTreeNode +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + fn new(is_leaf: bool) -> Self { + BPlusTreeNode { + is_leaf, + keys: vec![], + children: vec![], + values: vec![], + } + } + + fn is_overflow(&self, order: usize) -> bool { + self.keys.len() > order + } + + fn get_median_index(order: usize) -> usize { + order >> 1 + } + + fn find_leaf_entry<'a>(&self, node: &'a BPlusTreeNode) -> &'a K { + if node.is_leaf { + node.keys.get(0).unwrap() + } else { + let child = node.children.get(0).unwrap(); + self.find_leaf_entry(child) + } + } + + fn query(&self, k: &K) -> Option<&V> { + if self.is_leaf { + return match self.keys.binary_search(&k) { + Ok(idx) => self.values.get(idx), + Err(_) => None, + }; + } + let node = self.children.get(self.get_entry_index_upper_bound(k)).unwrap(); + node.query(k) + } + + fn get_equal_entry_index(&self, key: &K) -> Option { + let mut l = 0; + let mut r = self.keys.len() - 1; + while l <= r { + let mid = l + ((r - l) >> 1); + let mid_key = self.keys.get(mid).unwrap(); + if mid_key == key { + return Some(mid); + } else if mid_key > key { + r = mid - 1; + } else { + l = mid + 1; + } + } + return None; + } + + fn get_entry_index_upper_bound(&self, key: &K) -> usize { + let mut l = 0; + let mut r = self.keys.len(); + while l < r { + let mid = l + ((r - l) >> 1); + let mid_key = self.keys.get(mid).unwrap(); + if mid_key <= key { + l = mid + 1; + } else { + r = mid; + } + } + return l; + } + + fn insert(&mut self, k: K, v: V, inner_order: usize, leaf_order: usize) -> Option> { + if self.is_leaf { + if let Some(eq_entry_index) = self.get_equal_entry_index(&k) { + self.values.insert(eq_entry_index, v); + return None; + } + let pos = self.get_entry_index_upper_bound(&k); + self.keys.insert(pos, k); + self.values.insert(pos, v); + if self.is_overflow(leaf_order) { + return Some(self.split(leaf_order)); + } + } else { + let pos = self.get_entry_index_upper_bound(&k); + let child = self.children.get_mut(pos).unwrap(); + let node = child.insert(k, v, inner_order, leaf_order); + if node.is_some() { + let key = self.find_leaf_entry(node.as_ref().unwrap()); + let idx = self.get_entry_index_upper_bound(key); + self.keys.insert(idx, key.clone()); + self.children.insert(idx + 1, node.unwrap()); + if self.is_overflow(inner_order) { + return Some(self.split(inner_order)); + } + } + } + None + } + + fn split(&mut self, order: usize) -> BPlusTreeNode { + let median = BPlusTreeNode::::get_median_index(order); + if self.is_leaf { + let mut node = BPlusTreeNode::new(true); + node.keys = self.keys.split_off(median); + node.values = self.values.split_off(median); + node + } else { + let mut node = BPlusTreeNode::new(false); + node.keys = self.keys.split_off(median + 1); + node.children = self.children.split_off(median + 1); + self.children.push(node.children.get(0).unwrap().clone()); + node + } + } + + // pub(crate) fn traverse(&self, visit: &mut F) + // where + // F: FnMut(&BPlusTreeNode), + // { + // visit(self); + // self.children.iter().for_each(|child| child.traverse(visit)); + // } + + fn serialize_to_blocks(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { + let mut current_offset = offset; + let mut cursor = io::Cursor::new(&mut *buffer); + + cursor.write_all(if self.is_leaf { &[1u8] } else { &[0u8] }).expect("Failed to serialize node type"); + + let keys_encoded = bincode::serialize(&self.keys).expect("Failed to serialize keys"); + let keys_bytes = keys_encoded.len() as u32; + cursor.write_all(&keys_bytes.to_le_bytes()).expect("Failed to write keys length"); + cursor.write_all(&keys_encoded).expect("Failed to write keys"); + + if self.is_leaf { + let values_encoded = bincode::serialize(&self.values).expect("Failed to serialize values"); + cursor.write_all(&(values_encoded.len() as u32).to_le_bytes()).expect("Failed to write values length"); + cursor.write_all(&values_encoded).expect("Failed to write values"); + } + + let cursor_pos = cursor.position(); + cursor.flush().expect("failed to flush cursor"); + file.seek(SeekFrom::Start(offset))?; + file.write_all(&buffer)?; + current_offset += BLOCK_SIZE as u64; + + if !self.is_leaf { + let pointer_offset = offset + cursor_pos as u64; + let mut pointer = vec![]; + for child in &self.children { + pointer.push(current_offset); + current_offset = child.serialize_to_blocks(file, buffer, current_offset)?; + } + + let pointer_encoded = bincode::serialize(&pointer).expect("Failed to encode pointer"); + let pointer_bytes = pointer_encoded.len() as u32; + + file.seek(SeekFrom::Start(pointer_offset))?; + file.write_all(&pointer_bytes.to_le_bytes())?; + file.write_all(&pointer_encoded)?; + } + + Ok(current_offset) + } + + fn deserialize_from_blocks(file: &mut R, buffer: &mut Vec, offset: u64) -> io::Result { + file.seek(SeekFrom::Start(offset))?; + file.read_exact(buffer)?; + + let mut cursor = io::Cursor::new(&mut *buffer); + // Read the node type + let mut node_type = [0u8; 1]; + cursor.read_exact(&mut node_type)?; + let is_leaf = u8::from_le_bytes(node_type) == 1u8; + + // Deserialize keys + let mut length_bytes = [0u8; 4]; + cursor.read_exact(&mut length_bytes)?; + let keys_length = u32::from_le_bytes(length_bytes) as usize; + + let mut keys_buffer: Vec = vec![0; keys_length]; + cursor.read_exact(&mut keys_buffer)?; + let keys: Vec = bincode::deserialize_from(&*keys_buffer).expect("Failed to deserialize keys"); + + // Deserialize values if leaf node + let values = if is_leaf { + cursor.read_exact(&mut length_bytes)?; + let values_length = u32::from_le_bytes(length_bytes) as usize; + let mut values_buffer: Vec = vec![0; values_length]; + cursor.read_exact(&mut values_buffer)?; + let values: Vec = bincode::deserialize_from(&*values_buffer).expect("Failed to deserialize values"); + values + } else { + vec![] + }; + + // Deserialize children indices if internal node + let children = if !is_leaf { + cursor.read_exact(&mut length_bytes)?; + let pointers_length = u32::from_le_bytes(length_bytes) as usize; + let mut pointers_buffer: Vec = vec![0; pointers_length]; + cursor.read_exact(&mut pointers_buffer)?; + let pointers: Vec = bincode::deserialize_from(&*pointers_buffer).expect("Failed to deserialize pointers"); + pointers.iter().map(|pointer| { + BPlusTreeNode::::deserialize_from_blocks(file, buffer, *pointer).expect("failed to deserialize at offset {pointer}") + }).collect() + } else { + vec![] + }; + + Ok(BPlusTreeNode { + is_leaf, + keys, + values, + children, + }) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct BPlusTree { + root: BPlusTreeNode, + inner_order: usize, + leaf_order: usize, +} + +impl BPlusTree +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn new() -> Self { + let key_size = size_of::() + POINTER_SIZE + size_of::() + BINCODE_OVERHEAD; + let inner_order = BLOCK_SIZE / key_size; + let leaf_order = BLOCK_SIZE / (key_size + size_of::() + BINCODE_OVERHEAD); + BPlusTree { + root: BPlusTreeNode::::new(true), + inner_order, + leaf_order, + } + } + + fn new_with_root(root: BPlusTreeNode::) -> Self { + let key_size = size_of::() + POINTER_SIZE + size_of::() + BINCODE_OVERHEAD; + let inner_order = BLOCK_SIZE / key_size; + let leaf_order = BLOCK_SIZE / (key_size + size_of::() + BINCODE_OVERHEAD); + BPlusTree { + root, + inner_order, + leaf_order, + } + } + + pub(crate) fn insert(&mut self, key: K, value: V) { + if self.root.keys.len() == 0 { + self.root.keys.push(key); + self.root.values.push(value); + return; + } + + if let Some(node) = self.root.insert(key, value, self.inner_order, self.leaf_order) { + let child_key = if node.is_leaf { + node.keys.get(0).as_ref().unwrap() + } else { + node.find_leaf_entry(&node) + }; + + let mut new_root = BPlusTreeNode::::new(false); + new_root.keys.push(child_key.clone()); + new_root.children.push(std::mem::replace(&mut self.root, BPlusTreeNode::new(true))); // `true` als Beispiel für ein Blatt + new_root.children.push(node); + + self.root = new_root; + } + } + + pub(crate) fn query(&self, key: &K) -> Option<&V> { + self.root.query(key) + } + + pub(crate) fn serialize(&self, filename: &str) -> io::Result { + let mut file = OpenOptions::new().write(true).create(true).open(filename)?; + let mut buffer = vec![0u8; BLOCK_SIZE]; + self.root.serialize_to_blocks(&mut file, &mut buffer, 0u64) + } + + pub(crate) fn deserialize(filename: &str) -> io::Result { + let mut file = File::open(filename).expect("Failed to open file"); + let mut buffer = vec![0u8; BLOCK_SIZE]; + let root = BPlusTreeNode::deserialize_from_blocks(&mut file, &mut buffer, 0)?; + Ok(BPlusTree::new_with_root(root)) + } + + // pub(crate) fn traverse(&self, mut visit: F) + // where + // F: FnMut(&BPlusTreeNode), + // { + // self.root.traverse(&mut visit); + // } +} + +#[cfg(test)] +mod tests { + use std::io; + + use serde::{Deserialize, Serialize}; + + use crate::utils::bplustree::BPlusTree; + + // Example usage with a simple struct + #[derive(Serialize, Deserialize, Debug, Clone)] + struct Value { + id: u32, + data: String, + } + + #[test] + fn insert_test() -> io::Result<()> { + let mut tree = BPlusTree::::new(); + for i in 0u32..=500 { + tree.insert(i, format!("Entry {i}")); + } + + + // // Traverse the tree + // tree.traverse(|node| { + // println!("Node: {:?}", node); + // }); + + // Serialize the tree to a file + tree.serialize("/tmp/tree.bin")?; + + // Deserialize the tree from the file + tree = BPlusTree::::deserialize("/tmp/tree.bin")?; + + // Query the tree + for i in 0u32..=500 { + assert!(tree.query(&i).is_some(), "Entry {} not found", i); + } + + Ok(()) + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 43590d89b..982af072c 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -7,3 +7,4 @@ pub (crate) mod config_reader; pub (crate) mod default_utils; pub (crate) mod multi_file_reader; // pub (crate) mod file_reader; +pub (crate) mod bplustree; From 11b1ae1aee578f3d8274804dc6d1eee98215d26a Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 19:59:28 +0100 Subject: [PATCH 04/11] Added BPlusTreeQuery on file --- src/utils/bplustree.rs | 215 +++++++++++++++++++++++++++-------------- 1 file changed, 143 insertions(+), 72 deletions(-) diff --git a/src/utils/bplustree.rs b/src/utils/bplustree.rs index 6f6d86ad2..4b63a7b2b 100644 --- a/src/utils/bplustree.rs +++ b/src/utils/bplustree.rs @@ -1,25 +1,12 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::marker::PhantomData; use serde::{Deserialize, Serialize}; const BINCODE_OVERHEAD: usize = 4; const BLOCK_SIZE: usize = 4096; const POINTER_SIZE: usize = size_of::>(); -// -// #[derive(Serialize, Deserialize, Debug, Clone)] -// struct LeafKeyValue { -// key: K, -// value: V, -// } -// -// #[derive(Serialize, Deserialize, Debug, Clone)] -// struct InnerKeyValue { -// key: K, -// value: V, -// children: u64 -// } -// #[derive(Serialize, Deserialize, Debug, Clone)] struct BPlusTreeNode { @@ -34,6 +21,8 @@ where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { + + #[inline] fn new(is_leaf: bool) -> Self { BPlusTreeNode { is_leaf, @@ -43,10 +32,12 @@ where } } + #[inline] fn is_overflow(&self, order: usize) -> bool { self.keys.len() > order } + #[inline] fn get_median_index(order: usize) -> usize { order >> 1 } @@ -60,69 +51,71 @@ where } } - fn query(&self, k: &K) -> Option<&V> { + fn query(&self, key: &K) -> Option<&V> { if self.is_leaf { - return match self.keys.binary_search(&k) { + return match self.keys.binary_search(&key) { Ok(idx) => self.values.get(idx), Err(_) => None, }; } - let node = self.children.get(self.get_entry_index_upper_bound(k)).unwrap(); - node.query(k) + let node = self.children.get(self.get_entry_index_upper_bound(key)).unwrap(); + node.query(key) } - fn get_equal_entry_index(&self, key: &K) -> Option { - let mut l = 0; - let mut r = self.keys.len() - 1; - while l <= r { - let mid = l + ((r - l) >> 1); - let mid_key = self.keys.get(mid).unwrap(); + fn get_equal_entry_index(&self, key: &K) -> Option + where + K: Ord, + { + let mut left = 0; + let mut right = self.keys.len().checked_sub(1)?; + while left <= right { + let mid = left + ((right - left) >> 1); + let mid_key = &self.keys[mid]; if mid_key == key { return Some(mid); } else if mid_key > key { - r = mid - 1; + right = mid.checked_sub(1)?; } else { - l = mid + 1; + left = mid + 1; } } - return None; + None } fn get_entry_index_upper_bound(&self, key: &K) -> usize { - let mut l = 0; - let mut r = self.keys.len(); - while l < r { - let mid = l + ((r - l) >> 1); - let mid_key = self.keys.get(mid).unwrap(); - if mid_key <= key { - l = mid + 1; + let mut left = 0; + let mut right = self.keys.len(); + while left < right { + let mid = left + ((right - left) >> 1); + if &self.keys[mid] <= key { + left = mid + 1; } else { - r = mid; + right = mid; } } - return l; + left } - fn insert(&mut self, k: K, v: V, inner_order: usize, leaf_order: usize) -> Option> { + fn insert(&mut self, key: K, v: V, inner_order: usize, leaf_order: usize) -> Option> { if self.is_leaf { - if let Some(eq_entry_index) = self.get_equal_entry_index(&k) { + if let Some(eq_entry_index) = self.get_equal_entry_index(&key) { self.values.insert(eq_entry_index, v); return None; } - let pos = self.get_entry_index_upper_bound(&k); - self.keys.insert(pos, k); + let pos = self.get_entry_index_upper_bound(&key); + self.keys.insert(pos, key); self.values.insert(pos, v); if self.is_overflow(leaf_order) { return Some(self.split(leaf_order)); } } else { - let pos = self.get_entry_index_upper_bound(&k); + let pos = self.get_entry_index_upper_bound(&key); let child = self.children.get_mut(pos).unwrap(); - let node = child.insert(k, v, inner_order, leaf_order); + let node = child.insert(key, v, inner_order, leaf_order); if node.is_some() { - let key = self.find_leaf_entry(node.as_ref().unwrap()); - let idx = self.get_entry_index_upper_bound(key); - self.keys.insert(idx, key.clone()); + let leaf_key = self.find_leaf_entry(node.as_ref().unwrap()); + let idx = self.get_entry_index_upper_bound(leaf_key); + self.keys.insert(idx, leaf_key.clone()); self.children.insert(idx + 1, node.unwrap()); if self.is_overflow(inner_order) { return Some(self.split(inner_order)); @@ -160,21 +153,21 @@ where let mut current_offset = offset; let mut cursor = io::Cursor::new(&mut *buffer); - cursor.write_all(if self.is_leaf { &[1u8] } else { &[0u8] }).expect("Failed to serialize node type"); + cursor.write_all(if self.is_leaf { &[1u8] } else { &[0u8] })?; - let keys_encoded = bincode::serialize(&self.keys).expect("Failed to serialize keys"); + let keys_encoded = bincode::serialize(&self.keys).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; let keys_bytes = keys_encoded.len() as u32; - cursor.write_all(&keys_bytes.to_le_bytes()).expect("Failed to write keys length"); - cursor.write_all(&keys_encoded).expect("Failed to write keys"); + cursor.write_all(&keys_bytes.to_le_bytes())?; + cursor.write_all(&keys_encoded)?; if self.is_leaf { - let values_encoded = bincode::serialize(&self.values).expect("Failed to serialize values"); - cursor.write_all(&(values_encoded.len() as u32).to_le_bytes()).expect("Failed to write values length"); - cursor.write_all(&values_encoded).expect("Failed to write values"); + let values_encoded = bincode::serialize(&self.values).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + cursor.write_all(&(values_encoded.len() as u32).to_le_bytes())?; + cursor.write_all(&values_encoded)?; } let cursor_pos = cursor.position(); - cursor.flush().expect("failed to flush cursor"); + cursor.flush()?; file.seek(SeekFrom::Start(offset))?; file.write_all(&buffer)?; current_offset += BLOCK_SIZE as u64; @@ -182,12 +175,12 @@ where if !self.is_leaf { let pointer_offset = offset + cursor_pos as u64; let mut pointer = vec![]; - for child in &self.children { + for child in &self.children { pointer.push(current_offset); current_offset = child.serialize_to_blocks(file, buffer, current_offset)?; } - let pointer_encoded = bincode::serialize(&pointer).expect("Failed to encode pointer"); + let pointer_encoded = bincode::serialize(&pointer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; let pointer_bytes = pointer_encoded.len() as u32; file.seek(SeekFrom::Start(pointer_offset))?; @@ -198,7 +191,7 @@ where Ok(current_offset) } - fn deserialize_from_blocks(file: &mut R, buffer: &mut Vec, offset: u64) -> io::Result { + fn deserialize_from_blocks(file: &mut R, buffer: &mut Vec, offset: u64, nested: bool) -> io::Result<(Self, Option>)> { file.seek(SeekFrom::Start(offset))?; file.read_exact(buffer)?; @@ -215,7 +208,7 @@ where let mut keys_buffer: Vec = vec![0; keys_length]; cursor.read_exact(&mut keys_buffer)?; - let keys: Vec = bincode::deserialize_from(&*keys_buffer).expect("Failed to deserialize keys"); + let keys: Vec = bincode::deserialize_from(&*keys_buffer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; // Deserialize values if leaf node let values = if is_leaf { @@ -223,32 +216,43 @@ where let values_length = u32::from_le_bytes(length_bytes) as usize; let mut values_buffer: Vec = vec![0; values_length]; cursor.read_exact(&mut values_buffer)?; - let values: Vec = bincode::deserialize_from(&*values_buffer).expect("Failed to deserialize values"); + let values: Vec = bincode::deserialize_from(&*values_buffer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; values } else { vec![] }; // Deserialize children indices if internal node - let children = if !is_leaf { + let (children, children_pointer) = if !is_leaf { cursor.read_exact(&mut length_bytes)?; let pointers_length = u32::from_le_bytes(length_bytes) as usize; let mut pointers_buffer: Vec = vec![0; pointers_length]; cursor.read_exact(&mut pointers_buffer)?; - let pointers: Vec = bincode::deserialize_from(&*pointers_buffer).expect("Failed to deserialize pointers"); - pointers.iter().map(|pointer| { - BPlusTreeNode::::deserialize_from_blocks(file, buffer, *pointer).expect("failed to deserialize at offset {pointer}") - }).collect() + let pointers: Vec = bincode::deserialize_from(&*pointers_buffer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + if nested { + let nodes: Result>, io::Error> = pointers + .iter() + .map(|pointer| { + BPlusTreeNode::::deserialize_from_blocks(file, buffer, *pointer, nested) + .map(|(node, _)| node) + .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string())) + }) + .collect(); + + (nodes?, None) + } else { + (vec![], Some(pointers)) + } } else { - vec![] + (vec![], None) }; - Ok(BPlusTreeNode { + Ok((BPlusTreeNode { is_leaf, keys, values, children, - }) + }, children_pointer)) } } @@ -316,13 +320,15 @@ where pub(crate) fn serialize(&self, filename: &str) -> io::Result { let mut file = OpenOptions::new().write(true).create(true).open(filename)?; let mut buffer = vec![0u8; BLOCK_SIZE]; - self.root.serialize_to_blocks(&mut file, &mut buffer, 0u64) + let result = self.root.serialize_to_blocks(&mut file, &mut buffer, 0u64); + file.flush()?; + result } pub(crate) fn deserialize(filename: &str) -> io::Result { - let mut file = File::open(filename).expect("Failed to open file"); + let mut file = File::open(filename)?; let mut buffer = vec![0u8; BLOCK_SIZE]; - let root = BPlusTreeNode::deserialize_from_blocks(&mut file, &mut buffer, 0)?; + let (root, _) = BPlusTreeNode::deserialize_from_blocks(&mut file, &mut buffer, 0, true)?; Ok(BPlusTree::new_with_root(root)) } @@ -334,13 +340,67 @@ where // } } +pub(crate) struct BPlusTreeQuery { + file: File, + _marker_k: PhantomData, + _marker_v: PhantomData, +} + +impl BPlusTreeQuery +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn new(filename: &str) -> io::Result { + let file = File::open(filename)?; + Ok(BPlusTreeQuery { + file, + _marker_k: Default::default(), + _marker_v: Default::default(), + }) + } + + fn get_entry_index_upper_bound(keys: &Vec, key: &K) -> usize { + let mut left = 0; + let mut right = keys.len(); + while left < right { + let mid = left + ((right - left) >> 1); + if &keys[mid] <= key { + left = mid + 1; + } else { + right = mid; + } + } + left + } + + pub(crate) fn query(&mut self, key: &K) -> io::Result> { + let mut offset = 0; + let mut buffer = vec![0u8; BLOCK_SIZE]; + loop { + let (node, pointers) = + BPlusTreeNode::::deserialize_from_blocks(&mut self.file, &mut buffer, offset, false)?; + + if node.is_leaf { + return match node.keys.binary_search(key) { + Ok(idx) => Ok(node.values.get(idx).cloned()), + Err(_) => Ok(None), + }; + } + + let child_idx = BPlusTreeQuery::::get_entry_index_upper_bound(&node.keys, key); + offset = *pointers.unwrap().get(child_idx).unwrap(); + } + } +} + #[cfg(test)] mod tests { use std::io; use serde::{Deserialize, Serialize}; - use crate::utils::bplustree::BPlusTree; + use crate::utils::bplustree::{BPlusTree, BPlusTreeQuery}; // Example usage with a simple struct #[derive(Serialize, Deserialize, Debug, Clone)] @@ -356,7 +416,6 @@ mod tests { tree.insert(i, format!("Entry {i}")); } - // // Traverse the tree // tree.traverse(|node| { // println!("Node: {:?}", node); @@ -370,7 +429,19 @@ mod tests { // Query the tree for i in 0u32..=500 { - assert!(tree.query(&i).is_some(), "Entry {} not found", i); + let found = tree.query(&i); + assert!(found.is_some(), "Entry {} not found", i); + assert!(found.unwrap().eq(&format!("Entry {i}")), "Entry {} not found", i); + } + + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new("/tmp/tree.bin")?; + for i in 0u32..=500 { + let found = tree_query.query(&i); + assert!(found.is_ok(), "Query ok"); + let found_entry = found.unwrap(); // Unwrap once and store the Option + assert!(found_entry.is_some(), "Entry {} not found", i); + let entry = found_entry.unwrap(); + assert!(entry.eq(&format!("Entry {i}")), "Entry {} not found", i); } Ok(()) From 0b41768d3886f41c4c1a2f162a0fe55e2b3415e5 Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 20:16:44 +0100 Subject: [PATCH 05/11] Added some error checks --- src/utils/bplustree.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/utils/bplustree.rs b/src/utils/bplustree.rs index 4b63a7b2b..975e8f941 100644 --- a/src/utils/bplustree.rs +++ b/src/utils/bplustree.rs @@ -183,6 +183,7 @@ where let pointer_encoded = bincode::serialize(&pointer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; let pointer_bytes = pointer_encoded.len() as u32; + // got to pointer part of the own block and write the pointers of the children file.seek(SeekFrom::Start(pointer_offset))?; file.write_all(&pointer_bytes.to_le_bytes())?; file.write_all(&pointer_encoded)?; @@ -351,8 +352,21 @@ where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { + fn is_multiple_of_block_size(file: &File) -> io::Result { + let file_size = file.metadata()?.len(); // Get the file size in bytes + Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE + } + pub(crate) fn new(filename: &str) -> io::Result { let file = File::open(filename)?; + match BPlusTreeQuery::::is_multiple_of_block_size(&file) { + Ok(valid) => { + if !valid { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); + } + } + Err(err) => return Err(err) + } Ok(BPlusTreeQuery { file, _marker_k: Default::default(), @@ -437,7 +451,7 @@ mod tests { let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new("/tmp/tree.bin")?; for i in 0u32..=500 { let found = tree_query.query(&i); - assert!(found.is_ok(), "Query ok"); + assert!(found.is_ok(), "Query not ok"); let found_entry = found.unwrap(); // Unwrap once and store the Option assert!(found_entry.is_some(), "Entry {} not found", i); let entry = found_entry.unwrap(); From 797335c73d0bc665328a7fa6bb36ce1a5dfc583d Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 20:20:17 +0100 Subject: [PATCH 06/11] Rempoved Cursor with direct buffer access --- src/utils/bplustree.rs | 66 ++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/src/utils/bplustree.rs b/src/utils/bplustree.rs index 975e8f941..5769f8501 100644 --- a/src/utils/bplustree.rs +++ b/src/utils/bplustree.rs @@ -1,7 +1,6 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; - use serde::{Deserialize, Serialize}; const BINCODE_OVERHEAD: usize = 4; @@ -151,29 +150,37 @@ where fn serialize_to_blocks(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { let mut current_offset = offset; - let mut cursor = io::Cursor::new(&mut *buffer); + let buffer_slice = &mut buffer[..]; - cursor.write_all(if self.is_leaf { &[1u8] } else { &[0u8] })?; + // Write node type (leaf or internal) + buffer_slice[0] = if self.is_leaf { 1u8 } else { 0u8 }; + let mut write_pos = 1; + // Serialize and write keys let keys_encoded = bincode::serialize(&self.keys).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; let keys_bytes = keys_encoded.len() as u32; - cursor.write_all(&keys_bytes.to_le_bytes())?; - cursor.write_all(&keys_encoded)?; + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&keys_bytes.to_le_bytes()); + write_pos += 4; + buffer_slice[write_pos..write_pos + keys_encoded.len()].copy_from_slice(&keys_encoded); + write_pos += keys_encoded.len(); + // If leaf, serialize and write values if self.is_leaf { let values_encoded = bincode::serialize(&self.values).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; - cursor.write_all(&(values_encoded.len() as u32).to_le_bytes())?; - cursor.write_all(&values_encoded)?; + let values_bytes = values_encoded.len() as u32; + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&values_bytes.to_le_bytes()); + write_pos += 4; + buffer_slice[write_pos..write_pos + values_encoded.len()].copy_from_slice(&values_encoded); + write_pos += values_encoded.len(); } - let cursor_pos = cursor.position(); - cursor.flush()?; + // Write buffer to file file.seek(SeekFrom::Start(offset))?; - file.write_all(&buffer)?; + file.write_all(&buffer_slice[..BLOCK_SIZE])?; current_offset += BLOCK_SIZE as u64; if !self.is_leaf { - let pointer_offset = offset + cursor_pos as u64; + let pointer_offset = offset + write_pos as u64; let mut pointer = vec![]; for child in &self.children { pointer.push(current_offset); @@ -183,7 +190,6 @@ where let pointer_encoded = bincode::serialize(&pointer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; let pointer_bytes = pointer_encoded.len() as u32; - // got to pointer part of the own block and write the pointers of the children file.seek(SeekFrom::Start(pointer_offset))?; file.write_all(&pointer_bytes.to_le_bytes())?; file.write_all(&pointer_encoded)?; @@ -196,28 +202,22 @@ where file.seek(SeekFrom::Start(offset))?; file.read_exact(buffer)?; - let mut cursor = io::Cursor::new(&mut *buffer); - // Read the node type - let mut node_type = [0u8; 1]; - cursor.read_exact(&mut node_type)?; - let is_leaf = u8::from_le_bytes(node_type) == 1u8; + // Read the node type directly from buffer + let is_leaf = buffer[0] == 1u8; + let mut read_pos = 1; // Deserialize keys - let mut length_bytes = [0u8; 4]; - cursor.read_exact(&mut length_bytes)?; - let keys_length = u32::from_le_bytes(length_bytes) as usize; - - let mut keys_buffer: Vec = vec![0; keys_length]; - cursor.read_exact(&mut keys_buffer)?; - let keys: Vec = bincode::deserialize_from(&*keys_buffer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let keys_length = u32::from_le_bytes(buffer[read_pos..read_pos + 4].try_into().unwrap()) as usize; + read_pos += 4; + let keys: Vec = bincode::deserialize(&buffer[read_pos..read_pos + keys_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + read_pos += keys_length; // Deserialize values if leaf node let values = if is_leaf { - cursor.read_exact(&mut length_bytes)?; - let values_length = u32::from_le_bytes(length_bytes) as usize; - let mut values_buffer: Vec = vec![0; values_length]; - cursor.read_exact(&mut values_buffer)?; - let values: Vec = bincode::deserialize_from(&*values_buffer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let values_length = u32::from_le_bytes(buffer[read_pos..read_pos + 4].try_into().unwrap()) as usize; + read_pos += 4; + let values: Vec = bincode::deserialize(&buffer[read_pos..read_pos + values_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + read_pos += values_length; values } else { vec![] @@ -225,11 +225,9 @@ where // Deserialize children indices if internal node let (children, children_pointer) = if !is_leaf { - cursor.read_exact(&mut length_bytes)?; - let pointers_length = u32::from_le_bytes(length_bytes) as usize; - let mut pointers_buffer: Vec = vec![0; pointers_length]; - cursor.read_exact(&mut pointers_buffer)?; - let pointers: Vec = bincode::deserialize_from(&*pointers_buffer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let pointers_length = u32::from_le_bytes(buffer[read_pos..read_pos + 4].try_into().unwrap()) as usize; + read_pos += 4; + let pointers: Vec = bincode::deserialize(&buffer[read_pos..read_pos + pointers_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; if nested { let nodes: Result>, io::Error> = pointers .iter() From a3032005a478b8e69296e75c6b0f532cd52353a5 Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 20:41:07 +0100 Subject: [PATCH 07/11] some code optimizations --- src/utils/bplustree.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/utils/bplustree.rs b/src/utils/bplustree.rs index 5769f8501..21c7a77f1 100644 --- a/src/utils/bplustree.rs +++ b/src/utils/bplustree.rs @@ -1,3 +1,4 @@ +use std::array::TryFromSliceError; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; @@ -7,6 +8,17 @@ const BINCODE_OVERHEAD: usize = 4; const BLOCK_SIZE: usize = 4096; const POINTER_SIZE: usize = size_of::>(); +#[inline] +fn u32_from_bytes(bytes: &[u8]) -> io::Result { + Ok(u32::from_le_bytes(bytes.try_into().map_err(|e: TryFromSliceError| io::Error::new(io::ErrorKind::Other, e.to_string()))?)) +} + +#[inline] +fn bincode_serialize(value: &T) -> io::Result> +where T: serde::Serialize, { + bincode::serialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) +} + #[derive(Serialize, Deserialize, Debug, Clone)] struct BPlusTreeNode { keys: Vec, @@ -157,7 +169,7 @@ where let mut write_pos = 1; // Serialize and write keys - let keys_encoded = bincode::serialize(&self.keys).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let keys_encoded = bincode_serialize(&self.keys)?; let keys_bytes = keys_encoded.len() as u32; buffer_slice[write_pos..write_pos + 4].copy_from_slice(&keys_bytes.to_le_bytes()); write_pos += 4; @@ -166,7 +178,7 @@ where // If leaf, serialize and write values if self.is_leaf { - let values_encoded = bincode::serialize(&self.values).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let values_encoded = bincode_serialize(&self.values)?; let values_bytes = values_encoded.len() as u32; buffer_slice[write_pos..write_pos + 4].copy_from_slice(&values_bytes.to_le_bytes()); write_pos += 4; @@ -181,13 +193,13 @@ where if !self.is_leaf { let pointer_offset = offset + write_pos as u64; - let mut pointer = vec![]; + let mut pointer = Vec::with_capacity(self.children.len()); for child in &self.children { pointer.push(current_offset); current_offset = child.serialize_to_blocks(file, buffer, current_offset)?; } - let pointer_encoded = bincode::serialize(&pointer).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let pointer_encoded = bincode_serialize(&pointer)?; let pointer_bytes = pointer_encoded.len() as u32; file.seek(SeekFrom::Start(pointer_offset))?; @@ -207,14 +219,14 @@ where let mut read_pos = 1; // Deserialize keys - let keys_length = u32::from_le_bytes(buffer[read_pos..read_pos + 4].try_into().unwrap()) as usize; + let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; read_pos += 4; let keys: Vec = bincode::deserialize(&buffer[read_pos..read_pos + keys_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; read_pos += keys_length; // Deserialize values if leaf node let values = if is_leaf { - let values_length = u32::from_le_bytes(buffer[read_pos..read_pos + 4].try_into().unwrap()) as usize; + let values_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; read_pos += 4; let values: Vec = bincode::deserialize(&buffer[read_pos..read_pos + values_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; read_pos += values_length; @@ -225,7 +237,7 @@ where // Deserialize children indices if internal node let (children, children_pointer) = if !is_leaf { - let pointers_length = u32::from_le_bytes(buffer[read_pos..read_pos + 4].try_into().unwrap()) as usize; + let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; read_pos += 4; let pointers: Vec = bincode::deserialize(&buffer[read_pos..read_pos + pointers_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; if nested { @@ -290,7 +302,7 @@ where } pub(crate) fn insert(&mut self, key: K, value: V) { - if self.root.keys.len() == 0 { + if self.root.keys.is_empty() { self.root.keys.push(key); self.root.values.push(value); return; From e6148ae71e7e9958ce67940816b5e9bb39786713 Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 20:45:47 +0100 Subject: [PATCH 08/11] Tree test refactored --- src/utils/bplustree.rs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/utils/bplustree.rs b/src/utils/bplustree.rs index 21c7a77f1..9e816e714 100644 --- a/src/utils/bplustree.rs +++ b/src/utils/bplustree.rs @@ -427,17 +427,20 @@ mod tests { use crate::utils::bplustree::{BPlusTree, BPlusTreeQuery}; // Example usage with a simple struct - #[derive(Serialize, Deserialize, Debug, Clone)] - struct Value { + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + struct Record { id: u32, data: String, } #[test] fn insert_test() -> io::Result<()> { - let mut tree = BPlusTree::::new(); + let mut tree = BPlusTree::::new(); for i in 0u32..=500 { - tree.insert(i, format!("Entry {i}")); + tree.insert(i, Record { + id: i, + data: format!("Entry {i}"), + }); } // // Traverse the tree @@ -449,23 +452,29 @@ mod tests { tree.serialize("/tmp/tree.bin")?; // Deserialize the tree from the file - tree = BPlusTree::::deserialize("/tmp/tree.bin")?; + tree = BPlusTree::::deserialize("/tmp/tree.bin")?; // Query the tree for i in 0u32..=500 { let found = tree.query(&i); assert!(found.is_some(), "Entry {} not found", i); - assert!(found.unwrap().eq(&format!("Entry {i}")), "Entry {} not found", i); + assert!(found.unwrap().eq(&Record { + id: i, + data: format!("Entry {i}"), + }), "Entry {} not found", i); } - let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new("/tmp/tree.bin")?; + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new("/tmp/tree.bin")?; for i in 0u32..=500 { let found = tree_query.query(&i); assert!(found.is_ok(), "Query not ok"); let found_entry = found.unwrap(); // Unwrap once and store the Option assert!(found_entry.is_some(), "Entry {} not found", i); let entry = found_entry.unwrap(); - assert!(entry.eq(&format!("Entry {i}")), "Entry {} not found", i); + assert!(entry.eq(&Record { + id: i, + data: format!("Entry {i}"), + }), "Entry {} not found", i); } Ok(()) From 0b0bfce316c1be80243a7bcf7054e3a407e9950c Mon Sep 17 00:00:00 2001 From: euzu Date: Sun, 27 Oct 2024 21:25:57 +0100 Subject: [PATCH 09/11] virtual_id handling with bplustree --- src/{utils => repository}/bplustree.rs | 62 +++++----- src/repository/mod.rs | 3 +- src/repository/playlist_repository.rs | 5 +- src/repository/target_id_mapping_record.rs | 127 ++++++--------------- src/utils/mod.rs | 2 - 5 files changed, 73 insertions(+), 126 deletions(-) rename src/{utils => repository}/bplustree.rs (92%) diff --git a/src/utils/bplustree.rs b/src/repository/bplustree.rs similarity index 92% rename from src/utils/bplustree.rs rename to src/repository/bplustree.rs index 9e816e714..6690cbf4c 100644 --- a/src/utils/bplustree.rs +++ b/src/repository/bplustree.rs @@ -2,6 +2,8 @@ use std::array::TryFromSliceError; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; +use std::path::Path; + use serde::{Deserialize, Serialize}; const BINCODE_OVERHEAD: usize = 4; @@ -15,7 +17,9 @@ fn u32_from_bytes(bytes: &[u8]) -> io::Result { #[inline] fn bincode_serialize(value: &T) -> io::Result> -where T: serde::Serialize, { +where + T: serde::Serialize, +{ bincode::serialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) } @@ -32,7 +36,6 @@ where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { - #[inline] fn new(is_leaf: bool) -> Self { BPlusTreeNode { @@ -43,6 +46,10 @@ where } } + pub(crate) fn max_key(&self) -> Option<&K> { + self.keys.iter().max() + } + #[inline] fn is_overflow(&self, order: usize) -> bool { self.keys.len() > order @@ -152,13 +159,13 @@ where } } - // pub(crate) fn traverse(&self, visit: &mut F) - // where - // F: FnMut(&BPlusTreeNode), - // { - // visit(self); - // self.children.iter().for_each(|child| child.traverse(visit)); - // } + pub(crate) fn traverse(&self, visit: &mut F) + where + F: FnMut(&BPlusTreeNode), + { + visit(self); + self.children.iter().for_each(|child| child.traverse(visit)); + } fn serialize_to_blocks(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { let mut current_offset = offset; @@ -178,7 +185,7 @@ where // If leaf, serialize and write values if self.is_leaf { - let values_encoded = bincode_serialize(&self.values)?; + let values_encoded = bincode_serialize(&self.values)?; let values_bytes = values_encoded.len() as u32; buffer_slice[write_pos..write_pos + 4].copy_from_slice(&values_bytes.to_le_bytes()); write_pos += 4; @@ -199,7 +206,7 @@ where current_offset = child.serialize_to_blocks(file, buffer, current_offset)?; } - let pointer_encoded = bincode_serialize(&pointer)?; + let pointer_encoded = bincode_serialize(&pointer)?; let pointer_bytes = pointer_encoded.len() as u32; file.seek(SeekFrom::Start(pointer_offset))?; @@ -328,27 +335,27 @@ where self.root.query(key) } - pub(crate) fn serialize(&self, filename: &str) -> io::Result { - let mut file = OpenOptions::new().write(true).create(true).open(filename)?; + pub(crate) fn serialize(&self, filepath: &Path) -> io::Result { + let mut file = OpenOptions::new().write(true).create(true).open(filepath)?; let mut buffer = vec![0u8; BLOCK_SIZE]; let result = self.root.serialize_to_blocks(&mut file, &mut buffer, 0u64); file.flush()?; result } - pub(crate) fn deserialize(filename: &str) -> io::Result { - let mut file = File::open(filename)?; + pub(crate) fn deserialize(filepath: &Path) -> io::Result { + let mut file = File::open(filepath)?; let mut buffer = vec![0u8; BLOCK_SIZE]; let (root, _) = BPlusTreeNode::deserialize_from_blocks(&mut file, &mut buffer, 0, true)?; Ok(BPlusTree::new_with_root(root)) } - // pub(crate) fn traverse(&self, mut visit: F) - // where - // F: FnMut(&BPlusTreeNode), - // { - // self.root.traverse(&mut visit); - // } + pub(crate) fn traverse(&self, mut visit: F) + where + F: FnMut(&BPlusTreeNode), + { + self.root.traverse(&mut visit); + } } pub(crate) struct BPlusTreeQuery { @@ -367,8 +374,8 @@ where Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE } - pub(crate) fn new(filename: &str) -> io::Result { - let file = File::open(filename)?; + pub(crate) fn new(filepath: &Path) -> io::Result { + let file = File::open(filepath)?; match BPlusTreeQuery::::is_multiple_of_block_size(&file) { Ok(valid) => { if !valid { @@ -421,10 +428,11 @@ where #[cfg(test)] mod tests { use std::io; + use std::path::PathBuf; use serde::{Deserialize, Serialize}; - use crate::utils::bplustree::{BPlusTree, BPlusTreeQuery}; + use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; // Example usage with a simple struct #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -449,10 +457,10 @@ mod tests { // }); // Serialize the tree to a file - tree.serialize("/tmp/tree.bin")?; + tree.serialize(&PathBuf::from("/tmp/tree.bin"))?; // Deserialize the tree from the file - tree = BPlusTree::::deserialize("/tmp/tree.bin")?; + tree = BPlusTree::::deserialize(&PathBuf::from("/tmp/tree.bin"))?; // Query the tree for i in 0u32..=500 { @@ -464,7 +472,7 @@ mod tests { }), "Entry {} not found", i); } - let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new("/tmp/tree.bin")?; + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new(&PathBuf::from("/tmp/tree.bin"))?; for i in 0u32..=500 { let found = tree_query.query(&i); assert!(found.is_ok(), "Query not ok"); diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 8ffa7c0d8..27328b19d 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod storage; mod indexed_document_writer; mod indexed_document_reader; -pub(crate) mod target_id_mapping_record; \ No newline at end of file +pub(crate) mod target_id_mapping_record; // pub (crate) mod file_reader; +pub(crate) mod bplustree; diff --git a/src/repository/playlist_repository.rs b/src/repository/playlist_repository.rs index e7ded5468..622d71254 100644 --- a/src/repository/playlist_repository.rs +++ b/src/repository/playlist_repository.rs @@ -13,10 +13,9 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> target: &ConfigTarget, cfg: &Config) -> Result<(), Vec> { let mut errors = vec![]; - // TODO get previous virtual-ids and match them to the playlist items match ensure_target_storage_path(cfg, target.name.as_str()) { Ok(target_path) => { - let mut target_id_mapping = TargetIdMapping::from_path(&get_target_id_mapping_file(&target_path)); + let mut target_id_mapping = TargetIdMapping::new(get_target_id_mapping_file(&target_path)); for group in &mut *playlist { for channel in &group.channels { let mut header = channel.header.borrow_mut(); @@ -40,7 +39,7 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), } { Ok(()) => { - if let Err(err) = target_id_mapping.to_path(&target_path) { + if let Err(err) = target_id_mapping.persist() { errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); } if !playlist.is_empty() { diff --git a/src/repository/target_id_mapping_record.rs b/src/repository/target_id_mapping_record.rs index 803a04203..4f0388eb9 100644 --- a/src/repository/target_id_mapping_record.rs +++ b/src/repository/target_id_mapping_record.rs @@ -1,17 +1,18 @@ use std::cmp::max; -use std::collections::BTreeMap; use std::fs::File; use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write}; -use std::path::Path; -use std::rc::Rc; -use crate::model::playlist::{PlaylistItemType}; +use std::path::{Path, PathBuf}; +use serde::{Deserialize, Serialize}; + +use crate::model::playlist::PlaylistItemType; +use crate::repository::bplustree::BPlusTree; use crate::utils::file_utils; /** This file contains the provider id, the virtual id, and an uuid - */ +#[derive(Serialize, Deserialize, Clone)] pub(crate) struct TargetIdMappingRecord { pub provider_id: u32, pub virtual_id: u32, @@ -57,118 +58,58 @@ impl TargetIdMappingRecord { pub(crate) struct TargetIdMapping { dirty: bool, virtual_id_counter: u32, - by_provider_id: BTreeMap>, - by_virtual_id: BTreeMap>, - by_uuid: BTreeMap<[u8; 32], Rc>, - records: Vec>, - file: Option + by_virtual_id: BPlusTree, + path: PathBuf } impl TargetIdMapping { - pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32], item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { - self.dirty = true; - self.virtual_id_counter += 1; - self.insert(TargetIdMappingRecord { provider_id, virtual_id: self.virtual_id_counter, uuid, item_type: *item_type, parent_virtual_id}); - self.virtual_id_counter - } + pub(crate) fn new(path: PathBuf) -> Self { + let mut by_virtual_id: BPlusTree = match BPlusTree::::deserialize(&path) { + Ok(tree) => { + tree + } + _ => BPlusTree::::new() + }; - fn new(records: Vec>, file: Option) -> Self { - let mut by_provider_id: BTreeMap> = BTreeMap::new(); - let mut by_virtual_id: BTreeMap> = BTreeMap::new(); - let mut by_uuid: BTreeMap<[u8; 32], Rc> = BTreeMap::new(); let mut virtual_id_counter: u32 = 0; - for record in &records { - by_provider_id.insert(record.provider_id, Rc::clone(record)); - by_virtual_id.insert(record.virtual_id, Rc::clone(record)); - by_uuid.insert(record.uuid, Rc::clone(record)); - virtual_id_counter = max(record.virtual_id, virtual_id_counter); - } + by_virtual_id.traverse(|node| { + match node.max_key() { + None => {} + Some(max_value) => { + virtual_id_counter = max(virtual_id_counter, *max_value); + } + } + }); TargetIdMapping { dirty: false, virtual_id_counter, - by_provider_id, by_virtual_id, - by_uuid, - records, - file + path } } fn insert(&mut self, record: TargetIdMappingRecord) { - let provider_id = record.provider_id; let virtual_id = record.virtual_id; - let uuid = record.uuid; - let shared_record = Rc::new(record); - self.by_provider_id.insert(provider_id, Rc::clone(&shared_record)); - self.by_virtual_id.insert(virtual_id, Rc::clone(&shared_record)); - self.by_uuid.insert(uuid, Rc::clone(&shared_record)); - self.records.push(shared_record); + self.by_virtual_id.insert(virtual_id, record); } - pub(crate) fn get_by_provider_id(&self, provider_id: u32) -> Option<&Rc> { - self.by_provider_id.get(&provider_id) + pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32], item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { + self.dirty = true; + self.virtual_id_counter += 1; + self.insert(TargetIdMappingRecord { provider_id, virtual_id: self.virtual_id_counter, uuid, item_type: *item_type, parent_virtual_id }); + self.virtual_id_counter } - pub(crate) fn get_by_virtual_id(&self, virtual_id: u32) -> Option<&Rc> { - self.by_virtual_id.get(&virtual_id) + pub(crate) fn get_by_virtual_id(&self, virtual_id: u32) -> Option<&TargetIdMappingRecord> { + self.by_virtual_id.query(&virtual_id) } - pub(crate) fn get_by_uuid(&self, uuid: &[u8; 32]) -> Option<&Rc> { - self.by_uuid.get(uuid) - } - - - pub fn persist(&mut self) -> Result<(), Error> { - let mut file = match self.file.take() { - Some(file) => file, - None => return Err(Error::new(ErrorKind::NotFound, "No file given")), - }; - let result = self.to_file(&mut file); - self.file = Some(file); - result - } - - pub fn to_file(&mut self, file: &mut File) -> Result<(), Error> { + pub(crate) fn persist(&mut self) -> Result<(), Error> { if self.dirty { - for record in &self.records { - let bytes = record.to_bytes(); - if let Err(err) = file.write_all(&bytes) { - return Err(err); - } - } + self.by_virtual_id.serialize(&self.path)?; } self.dirty = false; Ok(()) } - pub fn to_path(&mut self, path: &Path) -> Result<(), Error> { - match file_utils::open_file_append(path, false) { - Ok(mut file) => self.to_file(&mut file), - Err(err) => Err(err) - } - } - - pub fn from_path(path: &Path) -> Self { - match file_utils::open_file_append(path, false) { - Ok(mut file) => { - let records = TargetIdMapping::read_records_from_file(&mut file); - TargetIdMapping::new(records, Some(file)) - }, - _ => TargetIdMapping::new(vec![], None) - } - } - - fn read_records_from_file(file: &mut File) -> Vec> { - let mut records = vec![]; - if let Ok(_) = file.seek(SeekFrom::Start(0)) { - let mut bytes = [0u8;45]; - loop { - if let Err(_) = file.read_exact(&mut bytes) { - break; - } - records.push(Rc::new(TargetIdMappingRecord::from_bytes(&bytes))); - } - } - records - } } \ No newline at end of file diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 982af072c..f8b5bcc91 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,5 +6,3 @@ pub (crate) mod json_utils; pub (crate) mod config_reader; pub (crate) mod default_utils; pub (crate) mod multi_file_reader; -// pub (crate) mod file_reader; -pub (crate) mod bplustree; From c904b725efd21ebc5b27c99285a85c5da2e85b4c Mon Sep 17 00:00:00 2001 From: euzu Date: Mon, 28 Oct 2024 20:15:27 +0100 Subject: [PATCH 10/11] virtual id mapping for m3u and xtream api --- Cargo.lock | 10 - Cargo.toml | 1 - src/api/m3u_api.rs | 2 - src/api/mod.rs | 2 +- src/api/xtream_api.rs | 101 +++----- src/model/config.rs | 7 + src/model/playlist.rs | 29 --- src/repository/bplustree.rs | 120 +++++---- src/repository/id_mapping.rs | 82 +++++++ src/repository/indexed_document_reader.rs | 111 ++++----- src/repository/indexed_document_writer.rs | 63 +++-- src/repository/m3u_repository.rs | 20 +- src/repository/mod.rs | 3 +- src/repository/playlist_repository.rs | 10 +- src/repository/target_id_mapping.rs | 102 ++++++++ src/repository/target_id_mapping_record.rs | 115 --------- src/repository/xtream_repository.rs | 267 ++++++++++----------- src/utils/file_utils.rs | 13 - src/utils/mod.rs | 2 +- test/rest-api.http | 9 + 20 files changed, 532 insertions(+), 537 deletions(-) create mode 100644 src/repository/id_mapping.rs create mode 100644 src/repository/target_id_mapping.rs delete mode 100644 src/repository/target_id_mapping_record.rs diff --git a/Cargo.lock b/Cargo.lock index f2f4e0b83..c4f8a86e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -602,15 +602,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" -[[package]] -name = "compressed_string" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3191fb9cd59eb0195c223d4f1d8894d2078957ee44f4940f059d11060a6d0e7" -dependencies = [ - "flate2", -] - [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1458,7 +1449,6 @@ dependencies = [ "blake3", "chrono", "clap", - "compressed_string", "cron", "enum-iterator", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index 2d1b392fc..8409d02a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,4 +50,3 @@ rpassword = "7.3" flate2 = "1" time = "0.3" blake3 = "1.5" -compressed_string = "1.0.0" diff --git a/src/api/m3u_api.rs b/src/api/m3u_api.rs index 1c2bf70c9..13fcda6ce 100644 --- a/src/api/m3u_api.rs +++ b/src/api/m3u_api.rs @@ -16,7 +16,6 @@ async fn m3u_api( //let api_req = UserApiRequest::from_map(&_api_req); match get_user_target(&api_req, &app_state) { Some((user, target)) => { - // let filename = target.get_m3u_filename(); if let Some(content) = m3u_load_rewrite_playlist(&app_state.config, target, &user) { HttpResponse::Ok().content_type(mime::TEXT_PLAIN_UTF_8).body(content) } else { @@ -39,7 +38,6 @@ async fn m3u_api_stream( if let Ok(m3u_stream_id) = stream_id.parse::() { if let Some((_user, target)) = get_user_target_by_credentials(&username, &password, &api_req, &app_state) { if target.has_output(&TargetType::M3u) { - match get_target_storage_path(&app_state.config, target.name.as_str()) { Some(target_path) => { if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) { diff --git a/src/api/mod.rs b/src/api/mod.rs index e18209461..9f97f7f37 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,4 +7,4 @@ mod xtream_api; mod m3u_api; mod xmltv_api; mod scheduler; -mod web_index; +mod web_index; \ No newline at end of file diff --git a/src/api/xtream_api.rs b/src/api/xtream_api.rs index e94b0da53..fa05a3c30 100644 --- a/src/api/xtream_api.rs +++ b/src/api/xtream_api.rs @@ -1,7 +1,6 @@ // https://github.com/tellytv/go.xtream-codes/blob/master/structs.go use std::collections::HashMap; -use std::convert::TryFrom; use std::fmt::{Display, Formatter}; use std::io::{Error, ErrorKind}; use std::path::Path; @@ -17,14 +16,11 @@ use crate::api::api_utils::{get_user_server_info, get_user_target, get_user_targ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigInput, ConfigTarget}; use crate::model::config::TargetType; -use crate::model::playlist::{PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; +use crate::model::playlist::{XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; -use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path, hash_string}; -use crate::repository::target_id_mapping_record::TargetIdMapping; use crate::repository::xtream_repository; use crate::utils::{json_utils, request_utils}; - enum XtreamApiStreamContext { LiveAlt, Live, @@ -283,45 +279,6 @@ fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.virtual_id))) } -fn get_and_write_xtream_series_info(config: &Config, target: &ConfigTarget, pli_series_info: &XtreamPlaylistItem, content: &str) -> Result { - if let Ok(mut doc) = serde_json::from_str::(content) { - if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { - let mut target_id_mapping = TargetIdMapping::from_path(&get_target_id_mapping_file(&target_path)); - - if let Some(episodes) = doc.get_mut("episodes") { - if let Some(episodes_map) = episodes.as_object_mut() { - let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); - for (_season, episode_list) in episodes_map { - // Iterate over items in the episode - if let Some(entries) = episode_list.as_array_mut() { - for entry in entries { - if let Some(episode) = entry.as_object_mut() { - if let Some(episode_id) = episode.get("id") { - if let Ok(provider_id) = episode_id.as_str().unwrap().parse::() { - let uuid_str = format!("{}/{}", pli_series_info.url, provider_id); - let uuid = hash_string(uuid_str.as_str()); - let virtual_id = target_id_mapping.insert_entry(provider_id, uuid, &PlaylistItemType::Series, pli_series_info.virtual_id); - episode.insert("id".to_string(), Value::String(virtual_id.to_string())); - } - } - if options.skip_series_direct_source { - episode.insert("direct_source".to_string(), Value::String(String::new())); - } - } - } - } - } - } - if let Ok(result) = serde_json::to_string(&doc) { - let _ = xtream_repository::xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result); - return Ok(result); - } - } - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli_series_info.virtual_id))) -} - async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> Result { request_utils::download_text_content(input, info_url, None).await } @@ -329,6 +286,7 @@ async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &ConfigTarget, pli: &XtreamPlaylistItem, info_url: &str, cluster: XtreamCluster) -> Result { if cluster == XtreamCluster::Series { + // TODO if expired then update content ! if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.as_str(), pli.virtual_id) { return Ok(content); } @@ -338,7 +296,7 @@ async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &C return match cluster { XtreamCluster::Live => Ok(content), XtreamCluster::Video => get_xtream_vod_info(target, pli, &content), - XtreamCluster::Series => get_and_write_xtream_series_info(config, target, pli, &content), + XtreamCluster::Series => xtream_repository::write_and_get_xtream_series_info(config, target, pli, &content), }; } @@ -377,12 +335,12 @@ async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserC async fn xtream_get_short_epg(app_state: &AppState, user: &ProxyUserCredentials, target: &ConfigTarget, stream_id: &str, limit: &str) -> HttpResponse { let target_name = &target.name; if target.has_output(&TargetType::Xtream) { - let req_stream_id: u32 = match FromStr::from_str(stream_id.trim()) { + let virtual_id: u32 = match FromStr::from_str(stream_id.trim()) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().finish() }; - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, None) { + if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None) { let input_id: u16 = pli.input_id; if let Some(input) = app_state.config.get_input_by_id(input_id) { if let Some(action_url) = get_xtream_player_api_action_url(input, "get_short_epg") { @@ -431,47 +389,48 @@ async fn xtream_player_api_handle_content_action(config: &Config, target_name: & } async fn xtream_get_catchup_response(app_state: &AppState, target: &ConfigTarget, stream_id: &str, start: &str, end: &str) -> HttpResponse { - let req_stream_id: u32 = match FromStr::from_str(stream_id) { + let virtual_id: u32 = match FromStr::from_str(stream_id) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().finish() }; - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, Some(XtreamCluster::Live)) { + !!!! why new ids for catchup ? + + if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(XtreamCluster::Live)) { let input_id = pli.input_id; if let Some(input) = app_state.config.get_input_by_id(input_id) { if let Some(info_url) = get_xtream_player_api_action_url(input, "get_simple_data_table") .map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id)) { if let Ok(content) = xtream_get_stream_info_content(info_url.as_str(), input).await { if let Ok(mut doc) = serde_json::from_str::>(content.as_str()) { - if let Some(epg_listings) = doc.get_mut("epg_listings") { - if let Some(epg_listing_list) = epg_listings.as_array_mut() { - let mapping = xtream_repository::xtream_load_catchup_id_mapping(&app_state.config, target.name.as_str()); - let mut max_id = u32::try_from(mapping.len()).unwrap(); - let mut new_id_mappings = Vec::new(); - for epg_list_value in epg_listing_list { - if let Some(epg_list_item) = epg_list_value.as_object_mut() { - // TODO epg_id - if let Some(Some(provider_id)) = epg_list_item.get("id").map(|v| v.as_str()) { - if let Ok(provider_stream_id) = &FromStr::from_str(provider_id) { - let stream_id = match mapping.get(provider_stream_id) { - None => { - max_id += 1; - new_id_mappings.push((*provider_stream_id, max_id)); - max_id - } - Some(mapped_id) => *mapped_id - }; - epg_list_item.insert("id".to_string(), Value::String(stream_id.to_string())); + if let Some(epg_listings) = doc.get_mut("epg_listings").and_then(|epg_listings| epg_listings.as_array_mut()) { + match xtream_repository::xtream_load_catchup_id_mapping(&app_state.config, target.name.as_str()) { + Ok(mut mapping) => { + let mut max_id = mapping.max_id(); + for epg_list_value in epg_listings { + if let Some(epg_list_item) = epg_list_value.as_object_mut() { + // TODO epg_id + if let Some(Some(provider_id)) = epg_list_item.get("id").map(|v| v.as_str()) { + if let Ok(provider_stream_id) = u32::from_str(provider_id) { + let stream_id = match mapping.query(provider_stream_id) { + None => { + max_id += 1; + mapping.insert(max_id, provider_stream_id); + max_id + } + Some(mapped_id) => *mapped_id + }; + epg_list_item.insert("id".to_string(), Value::String(stream_id.to_string())); + } } } } - } - if !new_id_mappings.is_empty() { - if let Err(err) = xtream_repository::xtream_write_catchup_id_mapping(&app_state.config, target.name.as_str(), &new_id_mappings) { + if let Err(err) = mapping.persist() { error!("Failed to write catchup id mapping {err}"); return HttpResponse::BadRequest().finish(); } } + Err(err) => { error!("Failed to load catchup id mapping {err}"); } } } diff --git a/src/model/config.rs b/src/model/config.rs index cfdfbf5e6..5761cc34b 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -42,6 +42,13 @@ macro_rules! valid_property { }}; } +#[macro_export] +macro_rules! create_m3u_filter_error { + ($kind: expr, $($arg:tt)*) => { + M3uFilterError::new($kind, format!($($arg)*)) + } +} + #[macro_export] macro_rules! create_m3u_filter_error_result { ($kind: expr, $($arg:tt)*) => { diff --git a/src/model/playlist.rs b/src/model/playlist.rs index 0d0e5abe7..87a67f096 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -44,18 +44,6 @@ pub(crate) enum XtreamCluster { } impl XtreamCluster { - // pub fn to_bytes(self) -> [u8; 1] { - // [self as u8] - // } - // - // pub fn from_bytes(bytes: [u8; 1]) -> Result { - // match bytes[0] { - // 1 => Ok(XtreamCluster::Live), - // 2 => Ok(XtreamCluster::Video), - // 3 => Ok(XtreamCluster::Series), - // _ => Err(format!("Invalid value for XtreamCluster: {}", bytes[0])), - // } - // } pub fn as_str(&self) -> &str { match self { @@ -99,23 +87,6 @@ pub(crate) enum PlaylistItemType { SeriesInfo = 4, } -impl PlaylistItemType { - pub fn to_bytes(self) -> [u8; 1] { - [self as u8] - } - - pub fn from_bytes(bytes: [u8; 1]) -> Result { - match bytes[0] { - 1 => Ok(PlaylistItemType::Live), - 2 => Ok(PlaylistItemType::Video), - 3 => Ok(PlaylistItemType::Series), - 4 => Ok(PlaylistItemType::SeriesInfo), - _ => Err(format!("Invalid value for PlaylistItemType: {}", bytes[0])), - } - } -} - - impl Default for PlaylistItemType { fn default() -> Self { PlaylistItemType::Live diff --git a/src/repository/bplustree.rs b/src/repository/bplustree.rs index 6690cbf4c..2dc7a1bf7 100644 --- a/src/repository/bplustree.rs +++ b/src/repository/bplustree.rs @@ -3,6 +3,8 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; use std::path::Path; +use flate2::Compression; +use log::error; use serde::{Deserialize, Serialize}; @@ -10,6 +12,11 @@ const BINCODE_OVERHEAD: usize = 4; const BLOCK_SIZE: usize = 4096; const POINTER_SIZE: usize = size_of::>(); +fn is_multiple_of_block_size(file: &File) -> io::Result { + let file_size = file.metadata()?.len(); // Get the file size in bytes + Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE +} + #[inline] fn u32_from_bytes(bytes: &[u8]) -> io::Result { Ok(u32::from_le_bytes(bytes.try_into().map_err(|e: TryFromSliceError| io::Error::new(io::ErrorKind::Other, e.to_string()))?)) @@ -23,6 +30,15 @@ where bincode::serialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) } +#[inline] +fn bincode_deserialize(value: &[u8]) -> io::Result +where + T: for<'a> serde::Deserialize<'a>, +{ + bincode::deserialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) +} + + #[derive(Serialize, Deserialize, Debug, Clone)] struct BPlusTreeNode { keys: Vec, @@ -46,10 +62,6 @@ where } } - pub(crate) fn max_key(&self) -> Option<&K> { - self.keys.iter().max() - } - #[inline] fn is_overflow(&self, order: usize) -> bool { self.keys.len() > order @@ -161,9 +173,11 @@ where pub(crate) fn traverse(&self, visit: &mut F) where - F: FnMut(&BPlusTreeNode), + F: FnMut(&Vec, &Vec), { - visit(self); + if self.is_leaf { + visit(&self.keys, &self.values); + } self.children.iter().for_each(|child| child.traverse(visit)); } @@ -177,25 +191,28 @@ where // Serialize and write keys let keys_encoded = bincode_serialize(&self.keys)?; - let keys_bytes = keys_encoded.len() as u32; - buffer_slice[write_pos..write_pos + 4].copy_from_slice(&keys_bytes.to_le_bytes()); + let keys_bytes_len = keys_encoded.len(); + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(keys_bytes_len as u32).to_le_bytes()); write_pos += 4; - buffer_slice[write_pos..write_pos + keys_encoded.len()].copy_from_slice(&keys_encoded); - write_pos += keys_encoded.len(); + buffer_slice[write_pos..write_pos + keys_bytes_len].copy_from_slice(&keys_encoded); + write_pos += keys_bytes_len; // If leaf, serialize and write values if self.is_leaf { let values_encoded = bincode_serialize(&self.values)?; - let values_bytes = values_encoded.len() as u32; - buffer_slice[write_pos..write_pos + 4].copy_from_slice(&values_bytes.to_le_bytes()); + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(&values_encoded)?; + let compressed_bytes = encoder.finish()?; + let values_bytes_len = compressed_bytes.len(); + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(values_bytes_len as u32).to_le_bytes()); write_pos += 4; - buffer_slice[write_pos..write_pos + values_encoded.len()].copy_from_slice(&values_encoded); - write_pos += values_encoded.len(); + buffer_slice[write_pos..write_pos + values_bytes_len].copy_from_slice(&compressed_bytes); + write_pos += values_bytes_len; } - // Write buffer to file + // Write the complete buffer to file, do not optimize to real filled size, file.seek(SeekFrom::Start(offset))?; - file.write_all(&buffer_slice[..BLOCK_SIZE])?; + file.write_all(&buffer_slice[..BLOCK_SIZE])?; // use BLOCK_SIZE current_offset += BLOCK_SIZE as u64; if !self.is_leaf { @@ -207,10 +224,10 @@ where } let pointer_encoded = bincode_serialize(&pointer)?; - let pointer_bytes = pointer_encoded.len() as u32; + let pointer_bytes_len = pointer_encoded.len() as u32; file.seek(SeekFrom::Start(pointer_offset))?; - file.write_all(&pointer_bytes.to_le_bytes())?; + file.write_all(&pointer_bytes_len.to_le_bytes())?; file.write_all(&pointer_encoded)?; } @@ -228,14 +245,18 @@ where // Deserialize keys let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; read_pos += 4; - let keys: Vec = bincode::deserialize(&buffer[read_pos..read_pos + keys_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let keys: Vec = bincode_deserialize(&buffer[read_pos..read_pos + keys_length])?; read_pos += keys_length; // Deserialize values if leaf node let values = if is_leaf { let values_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; read_pos += 4; - let values: Vec = bincode::deserialize(&buffer[read_pos..read_pos + values_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let compressed_bytes = &buffer[read_pos..read_pos + values_length]; + let mut decoder = flate2::write::ZlibDecoder::new(Vec::new()); + decoder.write_all(&compressed_bytes)?; + let values_bytes = decoder.finish()?; + let values: Vec = bincode_deserialize(&values_bytes)?; read_pos += values_length; values } else { @@ -246,7 +267,7 @@ where let (children, children_pointer) = if !is_leaf { let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; read_pos += 4; - let pointers: Vec = bincode::deserialize(&buffer[read_pos..read_pos + pointers_length]).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + let pointers: Vec = bincode_deserialize(&buffer[read_pos..read_pos + pointers_length])?; if nested { let nodes: Result>, io::Error> = pointers .iter() @@ -345,6 +366,9 @@ where pub(crate) fn deserialize(filepath: &Path) -> io::Result { let mut file = File::open(filepath)?; + if is_multiple_of_block_size(&file).is_err() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); + } let mut buffer = vec![0u8; BLOCK_SIZE]; let (root, _) = BPlusTreeNode::deserialize_from_blocks(&mut file, &mut buffer, 0, true)?; Ok(BPlusTree::new_with_root(root)) @@ -352,7 +376,7 @@ where pub(crate) fn traverse(&self, mut visit: F) where - F: FnMut(&BPlusTreeNode), + F: FnMut(&Vec, &Vec), { self.root.traverse(&mut visit); } @@ -369,14 +393,9 @@ where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { - fn is_multiple_of_block_size(file: &File) -> io::Result { - let file_size = file.metadata()?.len(); // Get the file size in bytes - Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE - } - - pub(crate) fn new(filepath: &Path) -> io::Result { + pub(crate) fn try_new(filepath: &Path) -> io::Result { let file = File::open(filepath)?; - match BPlusTreeQuery::::is_multiple_of_block_size(&file) { + match is_multiple_of_block_size(&file) { Ok(valid) => { if !valid { return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); @@ -405,22 +424,26 @@ where left } - pub(crate) fn query(&mut self, key: &K) -> io::Result> { + pub(crate) fn query(&mut self, key: &K) -> Option { let mut offset = 0; let mut buffer = vec![0u8; BLOCK_SIZE]; loop { - let (node, pointers) = - BPlusTreeNode::::deserialize_from_blocks(&mut self.file, &mut buffer, offset, false)?; - - if node.is_leaf { - return match node.keys.binary_search(key) { - Ok(idx) => Ok(node.values.get(idx).cloned()), - Err(_) => Ok(None), - }; - } - - let child_idx = BPlusTreeQuery::::get_entry_index_upper_bound(&node.keys, key); - offset = *pointers.unwrap().get(child_idx).unwrap(); + match BPlusTreeNode::::deserialize_from_blocks(&mut self.file, &mut buffer, offset, false) { + Ok((node, pointers)) => { + if node.is_leaf { + return match node.keys.binary_search(key) { + Ok(idx) => node.values.get(idx).cloned(), + Err(_) => None, + }; + } + let child_idx = BPlusTreeQuery::::get_entry_index_upper_bound(&node.keys, key); + offset = *pointers.unwrap().get(child_idx).unwrap(); + } + Err(err) => { + error!("Failed to read id tree from file {err}"); + return None; + } + }; } } } @@ -456,11 +479,12 @@ mod tests { // println!("Node: {:?}", node); // }); + let filepath = PathBuf::from("/tmp/tree.bin"); // Serialize the tree to a file - tree.serialize(&PathBuf::from("/tmp/tree.bin"))?; + tree.serialize(&filepath)?; // Deserialize the tree from the file - tree = BPlusTree::::deserialize(&PathBuf::from("/tmp/tree.bin"))?; + tree = BPlusTree::::deserialize(&filepath)?; // Query the tree for i in 0u32..=500 { @@ -472,13 +496,11 @@ mod tests { }), "Entry {} not found", i); } - let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::new(&PathBuf::from("/tmp/tree.bin"))?; + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::try_new(&filepath)?; for i in 0u32..=500 { let found = tree_query.query(&i); - assert!(found.is_ok(), "Query not ok"); - let found_entry = found.unwrap(); // Unwrap once and store the Option - assert!(found_entry.is_some(), "Entry {} not found", i); - let entry = found_entry.unwrap(); + assert!(found.is_some(), "Entry {} not found", i); + let entry = found.unwrap(); assert!(entry.eq(&Record { id: i, data: format!("Entry {i}"), diff --git a/src/repository/id_mapping.rs b/src/repository/id_mapping.rs new file mode 100644 index 000000000..7ad77478a --- /dev/null +++ b/src/repository/id_mapping.rs @@ -0,0 +1,82 @@ +use std::cmp::max; +use std::io::Error; +use std::path::{Path, PathBuf}; + +use log::error; +use serde::{Deserialize, Serialize}; + +use crate::repository::bplustree::BPlusTree; + +pub(crate) struct IdMapping +where + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + dirty: bool, + tree: BPlusTree, + path: PathBuf, + max_id: u32, +} + +impl IdMapping +where + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn new(path: &Path) -> Self { + let tree: BPlusTree = match BPlusTree::::deserialize(&path) { + Ok(tree) => tree, + _ => BPlusTree::::new() + }; + + let mut max_id = 0; + tree.traverse(|keys, _| { + match keys.iter().max() { + None => {} + Some(max_value) => { + max_id = max(max_id, *max_value); + } + } + }); + + IdMapping { + dirty: false, + tree, + path: path.to_path_buf(), + max_id, + } + } + + pub(crate) fn insert(&mut self, id: u32, value: V) { + self.dirty = true; + self.tree.insert(id, value); + } + + pub(crate) fn persist(&mut self) -> Result<(), Error> { + if self.dirty { + self.tree.serialize(&self.path)?; + } + self.dirty = false; + Ok(()) + } + + pub(crate) fn max_id(&self) -> u32 { + self.max_id + } + + pub(crate) fn query(&self, id: u32) -> Option<&V> { + self.tree.query(&id) + } +} + +impl Drop for IdMapping +where + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + fn drop(&mut self) { + match self.persist() { + Ok(_) => {} + Err(err) => { + error!("Failed to persist id mapping {:?} err:{}", &self.path, err.to_string()) + } + } + } +} \ No newline at end of file diff --git a/src/repository/indexed_document_reader.rs b/src/repository/indexed_document_reader.rs index 702786bc1..8c6d4a1c0 100644 --- a/src/repository/indexed_document_reader.rs +++ b/src/repository/indexed_document_reader.rs @@ -3,8 +3,20 @@ use std::fs::File; use std::io::{Error, ErrorKind, Read, Seek, SeekFrom}; use std::marker::PhantomData; use std::path::Path; -use crate::repository::indexed_document_writer::get_record_size; +use crate::repository::bplustree::BPlusTreeQuery; +use crate::repository::indexed_document_writer::OffsetPointer; +fn get_offset(index_path: &Path, doc_id: u32) -> Result { + match BPlusTreeQuery::::try_new(index_path) { + Ok(mut tree) => { + match tree.query(&doc_id) { + Some(offset) => Ok(offset as u64), + None => Err(Error::new(ErrorKind::NotFound, format!("doc_id not found {doc_id}"))), + } + }, + Err(err) => Err(err) + } +} fn read_content_size(main_file: &mut File) -> Result { let mut size_bytes = [0u8; 4]; @@ -23,36 +35,31 @@ pub(in crate::repository) struct IndexedDocumentReader { } impl IndexedDocumentReader { - pub fn new(main_path: &Path, index_path: &Path) -> Result, Error> { - if main_path.exists() && index_path.exists() { + pub fn new(main_path: &Path) -> Result, Error> { + if main_path.exists() { match File::open(main_path) { Ok(main_file) => { - match File::open(index_path) { - Ok(index_file) => { - let size = match index_file.metadata() { - Ok(metadata) => { - usize::try_from(metadata.len()).map_err(|err| Error::new(ErrorKind::Other, err))? - } - Err(_e) => 0, - }; - Ok(Self { - main_file, - cursor: 0, - size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?, - failed: false, - t_buffer: Vec::new(), - t_type: PhantomData, - }) + let size = match main_file.metadata() { + Ok(metadata) => { + usize::try_from(metadata.len()).map_err(|err| Error::new(ErrorKind::Other, err))? } - Err(e) => Err(e) - } + Err(_e) => 0, + }; + + Ok(Self { + main_file, + cursor: 0, + size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?, + failed: false, + t_buffer: Vec::new(), + t_type: PhantomData, + }) } Err(e) => Err(e) } } else { - Err(Error::new(ErrorKind::NotFound, format!("File not found {} or {}", - main_path.to_str().unwrap(), - index_path.to_str().unwrap()))) + Err(Error::new(ErrorKind::NotFound, format!("File not found {}", + main_path.to_str().unwrap()))) } } @@ -69,6 +76,7 @@ impl IndexedDocumentReader { } // read content-size let buf_size: usize = read_content_size(&mut self.main_file)?; + self.cursor += 4; // resize buffer if necessary if self.t_buffer.capacity() < buf_size { self.t_buffer.reserve(buf_size - self.t_buffer.capacity()); @@ -77,19 +85,34 @@ impl IndexedDocumentReader { // read content self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?; + self.cursor += buf_size as u32; // deserialize buffer return match bincode::deserialize::(&self.t_buffer[0..buf_size]) { - Ok(value) => { - self.cursor += get_record_size(); - Ok(Some(value)) - } + Ok(value) => Ok(Some(value)), Err(err) => { self.failed = true; Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))) } }; } + + pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, doc_id: u32) -> Result + { + if main_path.exists() && index_path.exists() { + // get the offset from index + let offset = crate::repository::indexed_document_reader::get_offset(index_path, doc_id)?; + let mut main_file = File::open(main_path)?; + main_file.seek(SeekFrom::Start(offset))?; + let buf_size = read_content_size(&mut main_file)?; + let mut buffer: Vec = vec![0; buf_size]; + main_file.read_exact(&mut buffer)?; + if let Ok(item) = bincode::deserialize::(&buffer) { + return Ok(item); + } + } + Err(Error::new(ErrorKind::Other, format!("Failed to read item for id {} - {}", doc_id, main_path.to_str().unwrap()))) + } } impl Iterator for IndexedDocumentReader { @@ -105,35 +128,3 @@ impl Iterator for IndexedDocumentReader None } } - -fn get_offset(file: &mut File, doc_id: u32) -> Result { - // TODO optimize this, sequential searching trough complete file! - let mut bytes = [0u8; 8]; - file.seek(SeekFrom::Start(0))?; - loop { - file.read_exact(&mut bytes)?; - let id = u32::from_le_bytes(bytes[..4].try_into().unwrap()); - if id == doc_id { - return Ok(u64::from(u32::from_le_bytes(bytes[4..].try_into().unwrap()))); - } - } -} - -pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, doc_id: u32) -> Result -where - T: ?Sized + serde::de::DeserializeOwned, -{ - if main_path.exists() && index_path.exists() { - let mut index_file = File::open(index_path)?; - let mut main_file = File::open(main_path)?; - let offset = get_offset(&mut index_file, doc_id)?; - main_file.seek(SeekFrom::Start(offset))?; - let buf_size = read_content_size(&mut main_file)?; - let mut buffer: Vec = vec![0; buf_size]; - main_file.read_exact(&mut buffer)?; - if let Ok(item) = bincode::deserialize::(&buffer[..]) { - return Ok(item); - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to read item for id {} - {}", doc_id, main_path.to_str().unwrap()))) -} diff --git a/src/repository/indexed_document_writer.rs b/src/repository/indexed_document_writer.rs index 2c55d6c2d..a98acae7f 100644 --- a/src/repository/indexed_document_writer.rs +++ b/src/repository/indexed_document_writer.rs @@ -2,20 +2,13 @@ use std::convert::TryFrom; use std::fs::File; use std::io::{Error, ErrorKind, Write}; use std::path::PathBuf; +use log::error; +use crate::repository::bplustree::BPlusTree; use crate::utils::file_utils; -use crate::utils::file_utils::create_file_tuple; +use crate::utils::file_utils::{open_file_append}; -pub(in crate::repository) fn get_record_size() -> u32 { 8 } - -fn to_bytes(left: u32, right: u32) -> [u8; 8] { - let left_bytes: [u8; 4] = left.to_le_bytes(); - let right_bytes: [u8; 4] = right.to_le_bytes(); - let mut combined_bytes: [u8; 8] = [0; 8]; - combined_bytes[..4].copy_from_slice(&left_bytes); - combined_bytes[4..].copy_from_slice(&right_bytes); - combined_bytes -} +pub(in crate::repository) type OffsetPointer = u32; /** * Creates two files, @@ -25,37 +18,40 @@ fn to_bytes(left: u32, right: u32) -> [u8; 8] { * Layout of content file record is: * - content-size (u32) + content * -* Layout of index file record is: -* - document_id (u32) + content file offset + (u32) +* index file is a bplustree */ pub(in crate::repository) struct IndexedDocumentWriter { main_path: PathBuf, index_path: PathBuf, main_file: File, - index_file: File, - main_offset: u32, - index_offset: u32, + main_offset: OffsetPointer, + index_tree: BPlusTree, } impl IndexedDocumentWriter { fn new_with_mode(main_path: PathBuf, index_path: PathBuf, append: bool) -> Result { - match create_file_tuple(&main_path, &index_path, append) { - Ok((main_file, index_file)) => { + match open_file_append(&main_path, append) { + Ok(main_file) => { let main_offset = match &main_file.metadata() { Ok(meta) => u32::try_from(meta.len()).map_err(|err| Error::new(ErrorKind::Other, err))?, Err(_) => 0 }; - let index_offset = match &index_file.metadata() { - Ok(meta) => u32::try_from(meta.len()).map_err(|err| Error::new(ErrorKind::Other, err))?, - Err(_) => 0 + + let index_tree = if append && index_path.exists() { + BPlusTree::::deserialize(&index_path).unwrap_or_else(|err| { + error!("Failed to load index {:?} {err}", index_path); + BPlusTree::::new() + }) + } else { + BPlusTree::::new() }; + Ok(Self { main_path, index_path, main_file, - index_file, main_offset, - index_offset, + index_tree }) } Err(e) => Err(e) @@ -70,30 +66,29 @@ impl IndexedDocumentWriter { Self::new_with_mode(main_path, index_path, true) } - pub fn write_doc(&mut self, doc_id: u32, doc: &T) -> Result<(u32, u32), Error> + pub fn flush(&mut self) -> std::io::Result<()> { + self.main_file.flush()?; + self.index_tree.serialize(&self.index_path).map(|_| ()) + } + + pub fn write_doc(&mut self, doc_id: u32, doc: &T) -> Result<(), Error> where T: ?Sized + serde::Serialize { - let current_main_index = self.main_offset; - let current_index_index = self.index_offset; if let Ok(encoded) = bincode::serialize(doc) { - let content_bytes = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; - let mut data: Vec = content_bytes.to_le_bytes().to_vec(); + let content_bytes_len = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; + let mut data: Vec = content_bytes_len.to_le_bytes().to_vec(); data.extend(&encoded); match file_utils::check_write(&self.main_file.write_all(&data)) { Ok(()) => { - let combined_bytes = to_bytes(doc_id, self.main_offset); - if let Err(err) = file_utils::check_write(&self.index_file.write_all(&combined_bytes)) { - return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.index_path.to_str().unwrap(), err))); - } + self.index_tree.insert(doc_id, self.main_offset); let written_bytes = u32::try_from(data.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; self.main_offset += written_bytes; - self.index_offset += get_record_size(); } Err(err) => { return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.main_path.to_str().unwrap(), err))); } } } - Ok((current_main_index, current_index_index)) + Ok(()) } } \ No newline at end of file diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 26fa249d7..5e1467612 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -4,20 +4,20 @@ use std::path::{Path, PathBuf}; use log::error; -use crate::{create_m3u_filter_error_result}; +use crate::{create_m3u_filter_error}; use crate::api::api_utils::get_user_server_info; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{M3uPlaylistItem, PlaylistGroup, PlaylistItem, PlaylistItemType}; -use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; +use crate::repository::indexed_document_reader::{IndexedDocumentReader}; use crate::repository::indexed_document_writer::IndexedDocumentWriter; use crate::repository::storage::{ensure_target_storage_path}; use crate::utils::file_utils; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err) + create_m3u_filter_error!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err) } } @@ -77,11 +77,12 @@ pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, target_pat for m3u in m3u_playlist { match writer.write_doc(m3u.virtual_id, &m3u) { Ok(_) => {}, - Err(err) => return cant_write_result!(&m3u_path, err) + Err(err) => return Err(cant_write_result!(&m3u_path, err)) } } + writer.flush().map_err(|err| cant_write_result!(&m3u_path, err))?; } - Err(err) => return cant_write_result!(&m3u_path, err) + Err(err) => return Err(cant_write_result!(&m3u_path, err)) } } } @@ -91,8 +92,8 @@ pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, target_pat pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, user: &ProxyUserCredentials) -> Option { match ensure_target_storage_path(cfg, target.name.as_str()) { Ok(target_path) => { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) { - match IndexedDocumentReader::::new(&m3u_path, &idx_path) { + if let Some((m3u_path, _)) = m3u_get_file_paths(&target_path) { + match IndexedDocumentReader::::new(&m3u_path) { Ok(mut reader) => { let server_info = get_user_server_info(cfg, user); let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); @@ -101,8 +102,7 @@ pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, use for m3u_pli in reader.by_ref() { match user.proxy { ProxyType::Reverse => { - let stream_id = m3u_pli.virtual_id; - result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{stream_id}").as_str()))); + result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{}", m3u_pli.virtual_id).as_str()))); } ProxyType::Redirect => { result.push(m3u_pli.to_m3u(target, None)); @@ -134,5 +134,5 @@ pub(crate) fn m3u_get_item_for_stream_id(stream_id: u32, m3u_path: &Path, idx_pa if stream_id < 1 { return Err(Error::new(ErrorKind::Other, "id should start with 1")); } - read_indexed_item::(m3u_path, idx_path, stream_id) + IndexedDocumentReader::::read_indexed_item(m3u_path, idx_path, stream_id) } \ No newline at end of file diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 27328b19d..91382caaf 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -7,5 +7,6 @@ pub(crate) mod storage; mod indexed_document_writer; mod indexed_document_reader; -pub(crate) mod target_id_mapping_record; // pub (crate) mod file_reader; +pub(crate) mod target_id_mapping; +pub(crate) mod id_mapping; pub(crate) mod bplustree; diff --git a/src/repository/playlist_repository.rs b/src/repository/playlist_repository.rs index 622d71254..38767d3b2 100644 --- a/src/repository/playlist_repository.rs +++ b/src/repository/playlist_repository.rs @@ -6,7 +6,7 @@ use crate::repository::epg_repository::epg_write; use crate::repository::kodi_repository::kodi_write_strm_playlist; use crate::repository::m3u_repository::m3u_write_playlist; use crate::repository::storage::{ensure_target_storage_path, get_target_id_mapping_file}; -use crate::repository::target_id_mapping_record::TargetIdMapping; +use crate::repository::target_id_mapping::TargetIdMapping; use crate::repository::xtream_repository::xtream_write_playlist; pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg>, @@ -15,7 +15,8 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> match ensure_target_storage_path(cfg, target.name.as_str()) { Ok(target_path) => { - let mut target_id_mapping = TargetIdMapping::new(get_target_id_mapping_file(&target_path)); + // reassign old virtual id or assign a new one + let mut target_id_mapping = TargetIdMapping::new(&get_target_id_mapping_file(&target_path)); for group in &mut *playlist { for channel in &group.channels { let mut header = channel.header.borrow_mut(); @@ -23,7 +24,10 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> Some(provider_id) => { let uuid = header.get_uuid(); let item_type = header.item_type; - header.virtual_id = target_id_mapping.insert_entry(provider_id, **uuid, &item_type, 0); + header.virtual_id = match target_id_mapping.get_by_uuid(&header.uuid) { + None => target_id_mapping.insert_entry(provider_id, **uuid, &item_type, 0), + Some(existing_id) => *existing_id + }; } None => { errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Playlistitem has no provider id: {}", &header.title))); diff --git a/src/repository/target_id_mapping.rs b/src/repository/target_id_mapping.rs new file mode 100644 index 000000000..2cbdecbbe --- /dev/null +++ b/src/repository/target_id_mapping.rs @@ -0,0 +1,102 @@ +use std::cmp::max; +use std::collections::BTreeMap; +use std::io::Error; +use std::path::{Path, PathBuf}; + +use chrono::Local; +use log::error; +use serde::{Deserialize, Serialize}; + +use crate::model::playlist::PlaylistItemType; +use crate::repository::bplustree::BPlusTree; + +const EXPIRATION_DURATION: i64 = 86400; + +#[derive(Serialize, Deserialize, Clone)] +pub(crate) struct VirtualIdRecord { + pub virtual_id: u32, + pub provider_id: u32, + pub uuid: [u8; 32], + pub item_type: PlaylistItemType, + pub parent_virtual_id: u32, // only for series to hold series info id. + pub last_updated: i64, +} + +impl VirtualIdRecord { + fn new(provider_id: u32, virtual_id: u32, item_type: PlaylistItemType, parent_virtual_id: u32, uuid: [u8; 32]) -> Self { + let last_updated = Local::now().timestamp(); + VirtualIdRecord { provider_id, virtual_id, uuid, item_type, parent_virtual_id, last_updated } + } + + pub(crate) fn is_expired(&self) -> bool { + (Local::now().timestamp() - self.last_updated) > EXPIRATION_DURATION + } +} + +pub(crate) struct TargetIdMapping { + dirty: bool, + virtual_id_counter: u32, + by_virtual_id: BPlusTree, + by_uuid: BTreeMap<[u8; 32], u32>, + path: PathBuf, +} + +impl TargetIdMapping { + pub(crate) fn new(path: &Path) -> Self { + let tree_virtual_id: BPlusTree = match BPlusTree::::deserialize(&path) { + Ok(tree) => tree, + _ => BPlusTree::::new() + }; + let mut tree_uuid = BTreeMap::new(); + let mut virtual_id_counter: u32 = 0; + tree_virtual_id.traverse(|keys, values| { + match keys.iter().max() { + None => {} + Some(max_value) => { + virtual_id_counter = max(virtual_id_counter, *max_value); + } + } + values.iter().for_each(|v| { + tree_uuid.insert(v.uuid, v.virtual_id); + }); + }); + TargetIdMapping { + dirty: false, + virtual_id_counter, + by_virtual_id: tree_virtual_id, + by_uuid: tree_uuid, + path: path.to_path_buf(), + } + } + + pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32], item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { + self.dirty = true; + self.virtual_id_counter += 1; + let record = VirtualIdRecord::new(provider_id, self.virtual_id_counter, *item_type, parent_virtual_id, uuid); + self.by_virtual_id.insert(self.virtual_id_counter, record); + self.virtual_id_counter + } + + pub(crate) fn get_by_uuid(&self, uuid: &[u8; 32]) -> Option<&u32> { + self.by_uuid.get(uuid) + } + + pub(crate) fn persist(&mut self) -> Result<(), Error> { + if self.dirty { + self.by_virtual_id.serialize(&self.path)?; + } + self.dirty = false; + Ok(()) + } +} + +impl Drop for TargetIdMapping { + fn drop(&mut self) { + match self.persist() { + Ok(_) => {} + Err(err) => { + error!("Failed to persist target id mapping {:?} err:{}", &self.path, err.to_string()) + } + } + } +} \ No newline at end of file diff --git a/src/repository/target_id_mapping_record.rs b/src/repository/target_id_mapping_record.rs deleted file mode 100644 index 4f0388eb9..000000000 --- a/src/repository/target_id_mapping_record.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::cmp::max; -use std::fs::File; -use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; - -use crate::model::playlist::PlaylistItemType; -use crate::repository::bplustree::BPlusTree; -use crate::utils::file_utils; - -/** -This file contains the provider id, the virtual id, and an uuid - */ -#[derive(Serialize, Deserialize, Clone)] -pub(crate) struct TargetIdMappingRecord { - pub provider_id: u32, - pub virtual_id: u32, - pub uuid: [u8; 32], - pub item_type: PlaylistItemType, - pub parent_virtual_id: u32, // only for series to hold series info id. -} - -impl TargetIdMappingRecord { - fn to_bytes(&self) -> [u8; 45] { - let provider_id_bytes: [u8; 4] = self.provider_id.to_le_bytes(); - let virtual_id_bytes: [u8; 4] = self.virtual_id.to_le_bytes(); - let parent_virtual_id_bytes: [u8; 4] = self.parent_virtual_id.to_le_bytes(); - let item_type_bytes: [u8; 1] = self.item_type.to_bytes(); - - let mut combined_bytes: [u8; 45] = [0; 45]; - combined_bytes[0..4].copy_from_slice(&provider_id_bytes); - combined_bytes[4..8].copy_from_slice(&virtual_id_bytes); - combined_bytes[8..12].copy_from_slice(&parent_virtual_id_bytes); - combined_bytes[12..13].copy_from_slice(&item_type_bytes); - combined_bytes[13..45].copy_from_slice(&self.uuid); - - combined_bytes - } - - fn from_bytes(bytes: &[u8; 45]) -> Self { - let provider_id_bytes: [u8; 4] = bytes[0..4].try_into().expect("Slice with incorrect length"); - let virtual_id_bytes: [u8; 4] = bytes[4..8].try_into().expect("Slice with incorrect length"); - let parent_virtual_id_bytes: [u8; 4] = bytes[8..12].try_into().expect("Slice with incorrect length"); - let item_type_bytes: [u8; 1] = bytes[12..13].try_into().expect("Slice with incorrect length"); - let uuid: [u8; 32] = bytes[13..45].try_into().expect("Slice with incorrect length"); - - TargetIdMappingRecord { - provider_id: u32::from_le_bytes(provider_id_bytes), - virtual_id: u32::from_le_bytes(virtual_id_bytes), - parent_virtual_id: u32::from_le_bytes(parent_virtual_id_bytes), - item_type: PlaylistItemType::from_bytes(item_type_bytes).unwrap(), - uuid, - } - } -} - -pub(crate) struct TargetIdMapping { - dirty: bool, - virtual_id_counter: u32, - by_virtual_id: BPlusTree, - path: PathBuf -} - -impl TargetIdMapping { - pub(crate) fn new(path: PathBuf) -> Self { - let mut by_virtual_id: BPlusTree = match BPlusTree::::deserialize(&path) { - Ok(tree) => { - tree - } - _ => BPlusTree::::new() - }; - - let mut virtual_id_counter: u32 = 0; - by_virtual_id.traverse(|node| { - match node.max_key() { - None => {} - Some(max_value) => { - virtual_id_counter = max(virtual_id_counter, *max_value); - } - } - }); - TargetIdMapping { - dirty: false, - virtual_id_counter, - by_virtual_id, - path - } - } - - fn insert(&mut self, record: TargetIdMappingRecord) { - let virtual_id = record.virtual_id; - self.by_virtual_id.insert(virtual_id, record); - } - - pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32], item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { - self.dirty = true; - self.virtual_id_counter += 1; - self.insert(TargetIdMappingRecord { provider_id, virtual_id: self.virtual_id_counter, uuid, item_type: *item_type, parent_virtual_id }); - self.virtual_id_counter - } - - pub(crate) fn get_by_virtual_id(&self, virtual_id: u32) -> Option<&TargetIdMappingRecord> { - self.by_virtual_id.query(&virtual_id) - } - - pub(crate) fn persist(&mut self) -> Result<(), Error> { - if self.dirty { - self.by_virtual_id.serialize(&self.path)?; - } - self.dirty = false; - Ok(()) - } - -} \ No newline at end of file diff --git a/src/repository/xtream_repository.rs b/src/repository/xtream_repository.rs index 70d9152d9..f03b97e34 100644 --- a/src/repository/xtream_repository.rs +++ b/src/repository/xtream_repository.rs @@ -1,22 +1,22 @@ use std::collections::HashMap; -use std::convert::TryInto; use std::fs::File; -use std::io::{BufReader, Error, ErrorKind, Read, Write}; +use std::io::{BufReader, Error, ErrorKind}; use std::path::{Path, PathBuf}; use log::error; use serde_json::{json, Value}; -use crate::create_m3u_filter_error_result; +use crate::{create_m3u_filter_error, create_m3u_filter_error_result}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; -use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; +use crate::repository::bplustree::{BPlusTreeQuery}; +use crate::repository::id_mapping::IdMapping; +use crate::repository::indexed_document_reader::{IndexedDocumentReader}; use crate::repository::indexed_document_writer::IndexedDocumentWriter; -use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path}; -use crate::repository::target_id_mapping_record::TargetIdMapping; -use crate::utils::file_utils; +use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path, hash_string}; +use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord}; use crate::utils::json_utils::{json_iter_array, json_write_documents_to_file}; pub(crate) static COL_CAT_LIVE: &str = "cat_live"; @@ -25,7 +25,7 @@ pub(crate) static COL_CAT_VOD: &str = "cat_vod"; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write xtream playlist: {} - {}", $path.to_str().unwrap() ,$err) + create_m3u_filter_error!(M3uFilterErrorKind::Notify, "failed to write xtream playlist: {} - {}", $path.to_str().unwrap() ,$err) } } @@ -33,23 +33,6 @@ fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } -// Maps episode id -> to provider_episode_id and series_id -fn xtream_get_series_episode_id_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("mapping_episode.db") -} - -// maps series_id to the index of series_info index -// direct access is not possible because the series_id is not ascending, it is random -fn xtream_get_series_id_series_info_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("mapping_series_info.db_idx") -} - -// maps catchup_id to provider_id -// direct access is not possible because the series_id is not ascending, it is random -fn xtream_get_catchup_id_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("mapping_catchup.db") -} - fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { if let Some(path) = xtream_get_storage_path(cfg, target_name) { if std::fs::create_dir_all(&path).is_err() { @@ -65,7 +48,7 @@ fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result Option<(PathBuf, PathBuf)> { if cluster == XtreamCluster::Series { - let xtream_path = storage_path.join("series_info.db"); + let xtream_path = storage_path.join("series_episodes.db"); let extension = xtream_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or(""))); let index_path = xtream_path.with_extension(format!("{}idx", &extension.unwrap_or_default())); return Some((xtream_path, index_path)); @@ -73,17 +56,8 @@ fn xtream_get_info_file_paths(storage_path: &Path, cluster: XtreamCluster) -> Op None } -fn xtream_clear_series_info(storage_path: &Path) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(storage_path, XtreamCluster::Series) { - let _ = std::fs::remove_file(info_path); - let _ = std::fs::remove_file(idx_path); - let _ = std::fs::remove_file(xtream_get_series_episode_id_mapping_file_path(storage_path)); - let _ = std::fs::remove_file(xtream_get_series_id_series_info_mapping_file_path(storage_path)); - } -} - -fn xtream_clear_catchup(storage_path: &Path) { - let _ = std::fs::remove_file(xtream_get_catchup_id_mapping_file_path(storage_path)); +fn xtream_get_catchup_id_mapping_file_path(storage_path: &Path) -> PathBuf { + storage_path.join("catchup_mapping.db") } fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { @@ -96,19 +70,15 @@ fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, Ok(xtream) => { match writer.write_doc(item.header.borrow().virtual_id, &xtream) { Ok(_) => {} - Err(err) => return cant_write_result!(&xtream_path, err) + Err(err) => return Err(cant_write_result!(&xtream_path, err)) } }, - Err(err) => return cant_write_result!(&xtream_path, err) + Err(err) => return Err(cant_write_result!(&xtream_path, err)) } } + writer.flush().map_err(|err| cant_write_result!(&xtream_path, err))?; } - Err(err) => return cant_write_result!(&xtream_path, err) - } - if cluster == XtreamCluster::Live { - xtream_clear_catchup(storage_path); - } else if cluster == XtreamCluster::Series { - xtream_clear_series_info(storage_path); + Err(err) => return Err(cant_write_result!(&xtream_path, err)) } } Ok(()) @@ -282,45 +252,64 @@ pub(crate) fn xtream_get_collection_path(cfg: &Config, target_name: &str, collec fn xtream_read_item_for_stream_id(stream_id: u32, storage_path: &Path, cluster: &XtreamCluster) -> Result { let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); - return read_indexed_item::(&xtream_path, &idx_path, stream_id); + return IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id); } fn xtream_read_series_item_for_stream_id(stream_id: u32, storage_path: &Path) -> Result { let (xtream_path, idx_path) = xtream_get_file_paths_for_series(storage_path); - return read_indexed_item::(&xtream_path, &idx_path, stream_id); + return IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id); } -pub(crate) fn xtream_get_item_for_stream_id(stream_id: u32, config: &Config, target: &ConfigTarget, xtream_cluster: Option) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { - let cluster = match xtream_cluster { - Some(c) => c, - None => { - if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { - let target_id_mapping = TargetIdMapping::from_path(&get_target_id_mapping_file(&target_path)); - if let Some(mapping) = target_id_mapping.get_by_virtual_id(stream_id) { - if mapping.item_type == PlaylistItemType::SeriesInfo { - return xtream_read_series_item_for_stream_id(stream_id, &storage_path); - } else { - XtreamCluster::try_from(mapping.item_type).unwrap() - } - } else { - return Err(Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {stream_id}"))); - } - } else { - return Err(Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name))); - } - } - }; - xtream_read_item_for_stream_id(stream_id, &storage_path, &cluster) +pub(crate) fn xtream_get_item_for_stream_id(virtual_id: u32, config: &Config, target: &ConfigTarget, xtream_cluster: Option) -> Result { + if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { + if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { + match BPlusTreeQuery::::try_new(&get_target_id_mapping_file(&target_path)) { + Ok(mut target_id_mapping) => { + return match target_id_mapping.query(&virtual_id) { + Some(mapping) => { + if mapping.item_type == PlaylistItemType::SeriesInfo { + xtream_read_series_item_for_stream_id(virtual_id, &storage_path) + } else if mapping.item_type == PlaylistItemType::Series && mapping.parent_virtual_id > 0 { + // we load the original series item + match xtream_read_series_item_for_stream_id(mapping.parent_virtual_id, &storage_path) { + Ok(mut item) => { + // we need to replace the provider id with the episode provider id + item.provider_id = mapping.provider_id; + Ok(item) + }, + Err(err) => Err(err) + } + } else { + let cluster = match xtream_cluster { + Some(c) => Some(c), + None => match XtreamCluster::try_from(mapping.item_type) { + Ok(item_type) => Some(item_type), + Err(_) => None + } + }; + match cluster { + Some(xc) => xtream_read_item_for_stream_id(virtual_id, &storage_path, &xc), + None => Err(Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {virtual_id}"))) + } + } + }, + None => Err(Error::new(ErrorKind::Other, format!("Could not find mappping for target {} and id {}", target.name, virtual_id))), + }; + }, + Err(err) => return Err(Error::new(ErrorKind::Other, format!("Could not load id mappping for target {} err:{}", target.name, err.to_string()))) + }; + } else { + return Err(Error::new(ErrorKind::Other, format!("Could not find path for target {} xtream output", &target.name))); + } } else { - Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}, storage path not found"))) + return Err(Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name))); } } pub(crate) fn xtream_load_rewrite_playlist(cluster: &XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result { if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { - let (xtream_path, idx_path) = xtream_get_file_paths(&storage_path, cluster); - match IndexedDocumentReader::::new(&xtream_path, &idx_path) { + let (xtream_path, _) = xtream_get_file_paths(&storage_path, cluster); + match IndexedDocumentReader::::new(&xtream_path) { Ok(mut reader) => { let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); let result: Vec = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id) @@ -347,9 +336,10 @@ pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, return match IndexedDocumentWriter::new_append(info_path.clone(), idx_path) { Ok(mut writer) => { match writer.write_doc(series_id, content) { - Ok(_) => Ok(()), + Ok(_) => {}, Err(_) => return Err(Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}"))) } + return Ok(writer.flush()?); } Err(err) => Err(err) }; @@ -361,82 +351,85 @@ pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, // Reads the series info entry if exists, otherwise error pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Result { if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path); - if series_id_index_mapping_path.exists() { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { - if info_path.exists() && idx_path.exists() { - let mut file = File::open(series_id_index_mapping_path)?; - let mut buffer = [0u8; 8]; - loop { - match file.read_exact(&mut buffer) { - Ok(()) => { - let stream_id = u32::from_le_bytes(buffer[..4].try_into().unwrap()); - if stream_id == series_id { - let index = u32::from_le_bytes(buffer[4..].try_into().unwrap()); - return read_indexed_item::(&info_path, &idx_path, index); - } - } - Err(e) => return Err(e), - } - } - } - } + if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { + if info_path.exists() && idx_path.exists() { + return IndexedDocumentReader::::read_indexed_item(&info_path, &idx_path, series_id); } } } Err(Error::new(ErrorKind::Other, format!("Failed to read series info for id {series_id} for {target_name}"))) } -pub(crate) fn xtream_write_catchup_id_mapping(config: &Config, target_name: &str, id_mappings: &Vec<(u32, u32)>) -> Result<(), Error> { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let file_path = xtream_get_catchup_id_mapping_file_path(&storage_path); - if let Ok(mut file) = if file_path.exists() { - std::fs::OpenOptions::new() - .append(true) // Open in append mode - .open(file_path) - } else { - File::create(file_path) - } { - for (provider_id, stream_id) in id_mappings { - let mut bytes: Vec = Vec::new(); - bytes.extend_from_slice(&provider_id.to_le_bytes()); - bytes.extend_from_slice(&stream_id.to_le_bytes()); - file_utils::check_write(&file.write_all(&bytes[..]))?; - } - return Ok(()); - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to get catchup info for {target_name}"))) -} - -// Returns hashmap with provider_id -> new_stream_id -pub(crate) fn xtream_load_catchup_id_mapping(config: &Config, target_name: &str) -> HashMap { - let mut result = HashMap::new(); +pub(crate) fn xtream_load_catchup_id_mapping(config: &Config, target_name: &str) -> Result, Error> { if let Some(storage_path) = xtream_get_storage_path(config, target_name) { let catchup_file = xtream_get_catchup_id_mapping_file_path(&storage_path); - if catchup_file.exists() { - if let Ok(file) = File::open(catchup_file) { - let mut reader = BufReader::new(&file); - let mut bytes = [0u8; 4]; - loop { - if reader.read_exact(&mut bytes).is_ok() { - let provider_id = u32::from_le_bytes(bytes); - let mut bytes = [0u8; 4]; - if reader.read_exact(&mut bytes).is_ok() { - let stream_id = u32::from_le_bytes(bytes); - result.insert(provider_id, stream_id); - } else { - break; + return Ok(IdMapping::::new(&catchup_file)); + } + Err(Error::new(ErrorKind::Other, format!("Failed to load catchup id mapping {target_name}"))) +} + +pub(crate) fn write_and_get_xtream_series_info(config: &Config, target: &ConfigTarget, pli_series_info: &XtreamPlaylistItem, content: &str) -> Result { + if let Ok(mut doc) = serde_json::from_str::(content) { + if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { + let mut target_id_mapping = TargetIdMapping::new(&get_target_id_mapping_file(&target_path)); + if let Some(episodes) = doc.get_mut("episodes").and_then(|e| e.as_object_mut()) { + let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); + for episode_list in episodes.values_mut() { + if let Some(entries) = episode_list.as_array_mut() { + for episode in entries.iter_mut().filter_map(|e| e.as_object_mut()) { + if let Some(episode_id) = episode.get("id").and_then(|id| id.as_str()) { + if let Ok(provider_id) = episode_id.parse::() { + let uuid = hash_string(&format!("{}/{}", pli_series_info.url, provider_id)); + let virtual_id = target_id_mapping.insert_entry(provider_id, uuid, &PlaylistItemType::Series, pli_series_info.virtual_id); + episode.insert("id".to_string(), Value::String(virtual_id.to_string())); + } + } + + if options.skip_series_direct_source { + episode.insert("direct_source".to_string(), Value::String(String::new())); + } } - } else { - break; } } + + drop(target_id_mapping); + if let Ok(result) = serde_json::to_string(&doc) { + let _ = xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result); + return Ok(result); + } } + + + // if let Some(episodes) = doc.get_mut("episodes") { + // if let Some(episodes_map) = episodes.as_object_mut() { + // let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); + // for (_season, episode_list) in episodes_map { + // // Iterate over items in the episode + // if let Some(entries) = episode_list.as_array_mut() { + // for entry in entries { + // if let Some(episode) = entry.as_object_mut() { + // if let Some(episode_id) = episode.get("id") { + // if let Ok(provider_id) = episode_id.as_str().unwrap().parse::() { + // let uuid = hash_string(format!("{}/{}", pli_series_info.url, provider_id).as_str()); + // let virtual_id = target_id_mapping.insert_entry(provider_id, uuid, &PlaylistItemType::Series, pli_series_info.virtual_id); + // episode.insert("id".to_string(), Value::String(virtual_id.to_string())); + // } + // } + // if options.skip_series_direct_source { + // episode.insert("direct_source".to_string(), Value::String(String::new())); + // } + // } + // } + // } + // } + // } + // drop(target_id_mapping); + // if let Ok(result) = serde_json::to_string(&doc) { + // let _ = xtream_repository::xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result); + // return Ok(result); + // } + // } } - } else { - error!("Failed to open catchup id-mapping file for {target_name}"); } - result + Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli_series_info.virtual_id))) } \ No newline at end of file diff --git a/src/utils/file_utils.rs b/src/utils/file_utils.rs index 21b7a84c1..d1e785445 100644 --- a/src/utils/file_utils.rs +++ b/src/utils/file_utils.rs @@ -196,16 +196,3 @@ pub(crate) fn open_file_append(path: &Path, append: bool) -> Result Result<(File, File), std::io::Error> { - match open_file_append(path1, append) { - Ok(file1) => { - match open_file_append(path2, append) { - Ok(file2) => Ok((file1, file2)), - Err(err) => - Err(std::io::Error::new(err.kind(), format!("failed to create file {} - {}", path2.to_str().unwrap(), err))) - } - } - Err(err) => - Err(std::io::Error::new(err.kind(), format!("failed to create file {} - {}", path1.to_str().unwrap(), err))) - } -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index f8b5bcc91..008bd750c 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,4 +5,4 @@ pub (crate) mod string_utils; pub (crate) mod json_utils; pub (crate) mod config_reader; pub (crate) mod default_utils; -pub (crate) mod multi_file_reader; +pub (crate) mod multi_file_reader; \ No newline at end of file diff --git a/test/rest-api.http b/test/rest-api.http index cb51d05e4..e7a59df00 100644 --- a/test/rest-api.http +++ b/test/rest-api.http @@ -24,3 +24,12 @@ GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&pas ### xtream series_categories GET http://localhost:8901/player_api.php?action=get_series_categories&username=xt&password=xt + +### xtream series +GET http://localhost:8901/player_api.php?action=get_series&username=xt&password=xt&category_id=120 + +### xtream series info +GET http://localhost:8901/player_api.php?action=get_series_info&username=xt&password=xt&series_id=16441 + +### xtream series stream + GET http://localhost:8901/series/xt/xt/18130 \ No newline at end of file From 02994fc906dfd3b9a9a550144e40cd14dd367c44 Mon Sep 17 00:00:00 2001 From: euzu Date: Tue, 29 Oct 2024 17:43:04 +0100 Subject: [PATCH 11/11] file locks for concurrent read write operations --- src/api/m3u_api.rs | 2 +- src/api/scheduler.rs | 68 ++++- src/api/xtream_api.rs | 178 ++++++------ src/filter.rs | 3 +- src/model/config.rs | 4 +- src/model/playlist.rs | 10 +- src/processing/playlist_processor.rs | 3 +- src/repository/bplustree.rs | 196 +++++++++---- src/repository/id_mapping.rs | 2 + src/repository/m3u_repository.rs | 85 +++--- src/repository/mod.rs | 1 - src/repository/playlist_repository.rs | 81 +++--- src/repository/storage.rs | 2 +- src/repository/target_id_mapping.rs | 25 +- src/repository/xtream_repository.rs | 382 ++++++++++++++------------ src/utils/file_lock_manager.rs | 106 +++++++ src/utils/mod.rs | 3 +- 17 files changed, 740 insertions(+), 411 deletions(-) create mode 100644 src/utils/file_lock_manager.rs diff --git a/src/api/m3u_api.rs b/src/api/m3u_api.rs index 13fcda6ce..fab63837f 100644 --- a/src/api/m3u_api.rs +++ b/src/api/m3u_api.rs @@ -41,7 +41,7 @@ async fn m3u_api_stream( match get_target_storage_path(&app_state.config, target.name.as_str()) { Some(target_path) => { if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) { - match m3u_get_item_for_stream_id(m3u_stream_id, &m3u_path, &idx_path) { + match m3u_get_item_for_stream_id(&app_state.config, m3u_stream_id, &m3u_path, &idx_path) { Ok(m3u_item) => { return stream_response(m3u_item.url.as_str(), &req, None).await; } diff --git a/src/api/scheduler.rs b/src/api/scheduler.rs index 4c0d0c6e1..a7286c739 100644 --- a/src/api/scheduler.rs +++ b/src/api/scheduler.rs @@ -1,29 +1,81 @@ use std::str::FromStr; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use actix_web::web::Data; -use chrono::Local; +use chrono::{DateTime, FixedOffset, Local}; use cron::Schedule; use log::error; use crate::api::api_model::AppState; use crate::exit; use crate::processing::playlist_processor::exec_processing; +fn datetime_to_instant(datetime: DateTime) -> Instant { + // Convert DateTime to SystemTime + let target_system_time: SystemTime = datetime.into(); + + // Get the current SystemTime + let now_system_time = SystemTime::now(); + + // Calculate the duration between now and the target time + let duration_until = target_system_time + .duration_since(now_system_time) + .unwrap_or_else(|_| Duration::from_secs(0)); + + // Get the current Instant and add the duration to calculate the target Instant + Instant::now() + duration_until +} + pub(crate) async fn start_scheduler(expression: &str, data: Data) -> ! { match Schedule::from_str(expression) { Ok(schedule) => { let offset = *Local::now().offset(); loop { let mut upcoming = schedule.upcoming(offset).take(1); - actix_rt::time::sleep(Duration::from_millis(500)).await; - let local = &Local::now(); - if let Some(datetime) = upcoming.next() { - if datetime.timestamp() <= local.timestamp() { - exec_processing(data.config.clone(), data.targets.clone()).await; - } + actix_rt::time::sleep_until(actix_rt::time::Instant::from(datetime_to_instant(datetime))).await; + exec_processing(data.config.clone(), data.targets.clone()).await; } } } Err(err) => exit!("Failed to start scheduler: {}", err) } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use std::sync::atomic::{AtomicU8, Ordering}; + use chrono::Local; + use cron::Schedule; + use crate::api::scheduler::datetime_to_instant; + + #[actix_rt::test] + async fn test_run_scheduler() { + // Define a cron expression that runs every second + let expression = "0/1 * * * * * *"; // every second + + let runs = AtomicU8::new(0); + let run_me = || runs.fetch_add(1, Ordering::Relaxed); + + let start = std::time::Instant::now(); + match Schedule::from_str(expression) { + Ok(schedule) => { + let offset = *Local::now().offset(); + loop { + let mut upcoming = schedule.upcoming(offset).take(1); + if let Some(datetime) = upcoming.next() { + actix_rt::time::sleep_until(actix_rt::time::Instant::from(datetime_to_instant(datetime))).await; + run_me(); + } + if runs.load(Ordering::Relaxed) == 6 { + break; + } + } + } + Err(_) => {} + }; + let duration = start.elapsed(); + + assert!(runs.load(Ordering::Relaxed) == 6, "Failed to run"); + assert!(duration.as_secs() > 4, "Failed time"); + } } \ No newline at end of file diff --git a/src/api/xtream_api.rs b/src/api/xtream_api.rs index fa05a3c30..e095affaf 100644 --- a/src/api/xtream_api.rs +++ b/src/api/xtream_api.rs @@ -16,11 +16,49 @@ use crate::api::api_utils::{get_user_server_info, get_user_target, get_user_targ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigInput, ConfigTarget}; use crate::model::config::TargetType; -use crate::model::playlist::{XtreamCluster, XtreamPlaylistItem}; +use crate::model::playlist::{PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; +use crate::repository::storage::{get_target_storage_path, hash_string}; +use crate::repository::target_id_mapping::TargetIdMapping; use crate::repository::xtream_repository; use crate::utils::{json_utils, request_utils}; + +macro_rules! try_option_bad_request { + ($option:expr, $msg_is_error:expr, $msg:expr) => { + match $option { + Some(value) => value, + None => { + if $msg_is_error {error!("{}", $msg);} else {debug!("{}", $msg);} + return HttpResponse::BadRequest().finish(); + } + } + }; + ($option:expr) => { + match $option { + Some(value) => value, + None => return HttpResponse::BadRequest().finish(), + } + }; +} +macro_rules! try_result_bad_request { + ($option:expr, $msg_is_error:expr, $msg:expr) => { + match $option { + Ok(value) => value, + Err(_) => { + if $msg_is_error {error!("{}", $msg);} else {debug!("{}", $msg);} + return HttpResponse::BadRequest().finish(); + } + } + }; + ($option:expr) => { + match $option { + Ok(value) => value, + Err(_) => return HttpResponse::BadRequest().finish(), + } + }; +} + enum XtreamApiStreamContext { LiveAlt, Live, @@ -107,7 +145,6 @@ fn get_xtream_player_api_stream_url(input: &ConfigInput, context: &str, action_p } } - fn get_user_info(user: &ProxyUserCredentials, cfg: &Config) -> XtreamAuthorizationResponse { let server_info = get_user_server_info(cfg, user); @@ -155,43 +192,34 @@ async fn xtream_player_api_stream( app_state: &web::Data, stream_req: XtreamApiStreamRequest<'_>, ) -> HttpResponse { - if let Some((user, target)) = get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state) { - let target_name = &target.name; - if target.has_output(&TargetType::Xtream) { - let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id); - let virtual_id: u32 = match FromStr::from_str(action_stream_id.trim()) { - Ok(id) => id, - Err(_) => return HttpResponse::BadRequest().finish() - }; - - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None) { - let input_id: u16 = pli.input_id; - if let Some(input) = app_state.config.get_input_by_id(input_id) { - let mut query_path = if stream_req.action_path.is_empty() { String::new() } else { format!("{}/", stream_req.action_path) }; - query_path = format!("{query_path}{}{stream_ext}", pli.provider_id); - if let Some(stream_url) = get_xtream_player_api_stream_url(input, stream_req.context.to_string().as_str(), query_path.as_str()) { - if user.proxy == ProxyType::Redirect { - debug!("Redirecting stream request to {stream_url}"); - return HttpResponse::Found().insert_header(("Location", stream_url)).finish(); - } - return stream_response(&stream_url, req, Some(input)).await; - } - error!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context); - } else { - error!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context); - } - } else { - error!("Failed to read xtream item for stream id {}", virtual_id); - } - } else { - debug!("Target has no xtream output {}", target_name); - } - } else { - debug!("Could not find any user {}", stream_req.username); + let (user, target) = try_option_bad_request!(get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state), false, format!("Could not find any user {}", stream_req.username)); + let target_name = &target.name; + if !target.has_output(&TargetType::Xtream) { + debug!("Target has no xtream output {}", target_name); + return HttpResponse::BadRequest().finish(); } - HttpResponse::BadRequest().finish() + let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id); + let virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); + let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None), true, format!("Failed to read xtream item for stream id {}", virtual_id)); + let input = try_option_bad_request!(app_state.config.get_input_by_id(pli.input_id), true, format!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context)); + + let query_path = if stream_req.action_path.is_empty() { + format!("{}{stream_ext}", pli.provider_id) + } else { + format!("{}/{}{stream_ext}", stream_req.action_path, pli.provider_id) + }; + + let stream_url = try_option_bad_request!(get_xtream_player_api_stream_url(input, stream_req.context.to_string().as_str(), &query_path), true, format!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context)); + + if user.proxy == ProxyType::Redirect { + debug!("Redirecting stream request to {stream_url}"); + return HttpResponse::Found().insert_header(("Location", stream_url)).finish(); + } + + stream_response(&stream_url, req, Some(input)).await } + async fn xtream_player_api_live_stream( req: HttpRequest, api_req: web::Query, @@ -286,8 +314,7 @@ async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &ConfigTarget, pli: &XtreamPlaylistItem, info_url: &str, cluster: XtreamCluster) -> Result { if cluster == XtreamCluster::Series { - // TODO if expired then update content ! - if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.as_str(), pli.virtual_id) { + if let Some(content) = xtream_repository::xtream_load_series_info(config, target.name.as_str(), pli.virtual_id) { return Ok(content); } } @@ -389,60 +416,33 @@ async fn xtream_player_api_handle_content_action(config: &Config, target_name: & } async fn xtream_get_catchup_response(app_state: &AppState, target: &ConfigTarget, stream_id: &str, start: &str, end: &str) -> HttpResponse { - let virtual_id: u32 = match FromStr::from_str(stream_id) { - Ok(id) => id, - Err(_) => return HttpResponse::BadRequest().finish() - }; + let virtual_id: u32 = try_result_bad_request!(FromStr::from_str(stream_id)); + let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(XtreamCluster::Live))); + let input = try_option_bad_request!(app_state.config.get_input_by_id(pli.input_id)); + let info_url = try_option_bad_request!(get_xtream_player_api_action_url(input, "get_simple_data_table").map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id))); + let content = try_result_bad_request!(xtream_get_stream_info_content(info_url.as_str(), input).await); + let mut doc: Map = try_result_bad_request!(serde_json::from_str(&content)); + let epg_listings = try_option_bad_request!(doc.get_mut("epg_listings").and_then(Value::as_array_mut)); + let target_path = try_option_bad_request!(get_target_storage_path(&app_state.config, target.name.as_str())); + let mut target_id_mapping = TargetIdMapping::new(&target_path); - !!!! why new ids for catchup ? - - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(XtreamCluster::Live)) { - let input_id = pli.input_id; - if let Some(input) = app_state.config.get_input_by_id(input_id) { - if let Some(info_url) = get_xtream_player_api_action_url(input, "get_simple_data_table") - .map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id)) { - if let Ok(content) = xtream_get_stream_info_content(info_url.as_str(), input).await { - if let Ok(mut doc) = serde_json::from_str::>(content.as_str()) { - if let Some(epg_listings) = doc.get_mut("epg_listings").and_then(|epg_listings| epg_listings.as_array_mut()) { - match xtream_repository::xtream_load_catchup_id_mapping(&app_state.config, target.name.as_str()) { - Ok(mut mapping) => { - let mut max_id = mapping.max_id(); - for epg_list_value in epg_listings { - if let Some(epg_list_item) = epg_list_value.as_object_mut() { - // TODO epg_id - if let Some(Some(provider_id)) = epg_list_item.get("id").map(|v| v.as_str()) { - if let Ok(provider_stream_id) = u32::from_str(provider_id) { - let stream_id = match mapping.query(provider_stream_id) { - None => { - max_id += 1; - mapping.insert(max_id, provider_stream_id); - max_id - } - Some(mapped_id) => *mapped_id - }; - epg_list_item.insert("id".to_string(), Value::String(stream_id.to_string())); - } - } - } - } - if let Err(err) = mapping.persist() { - error!("Failed to write catchup id mapping {err}"); - return HttpResponse::BadRequest().finish(); - } - } - Err(err) => { error!("Failed to load catchup id mapping {err}"); } - } - } - - if let Ok(result) = serde_json::to_string(&doc) { - return HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(result); - } - } - } - } + for epg_list_item in epg_listings.iter_mut().filter_map(Value::as_object_mut) { + // TODO epg_id + if let Some(catchup_provider_id) = epg_list_item.get("id").and_then(Value::as_str).and_then(|id| id.parse::().ok()) { + let uuid = hash_string(&format!("{}/{}", pli.url, catchup_provider_id)); + let virtual_id = target_id_mapping.insert_entry(uuid, catchup_provider_id, &PlaylistItemType::Catchup, pli.provider_id); + epg_list_item.insert("id".to_string(), Value::String(virtual_id.to_string())); } } - HttpResponse::BadRequest().finish() + if let Err(err) = target_id_mapping.persist() { + error!("Failed to write catchup id mapping {err}"); + return HttpResponse::BadRequest().finish(); + } + + match serde_json::to_string(&doc) { + Ok(result) => HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(result), + Err(_) => HttpResponse::BadRequest().finish(), + } } async fn xtream_player_api( diff --git a/src/filter.rs b/src/filter.rs index 2502d57bb..0e6b1141e 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -196,7 +196,8 @@ impl std::fmt::Display for Filter { PlaylistItemType::Live => "live", PlaylistItemType::Video => "vod", PlaylistItemType::Series => "series", - PlaylistItemType::SeriesInfo => "series" // yes series-info is handled as series in filter + PlaylistItemType::SeriesInfo => "series", // yes series-info is handled as series in filter + _ => "unsupported" }) } Filter::Group(stmt) => { diff --git a/src/model/config.rs b/src/model/config.rs index 5761cc34b..1e9f84b4f 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -24,6 +24,7 @@ use crate::utils::{config_reader, file_utils}; use crate::utils::default_utils::{default_as_default, default_as_false, default_as_true, default_as_empty_list, default_as_frm, default_as_empty_map, default_as_zero_u8, default_as_two_u16}; +use crate::utils::file_lock_manager::FileLockManager; pub(crate) const MAPPER_ATTRIBUTE_FIELDS: &[&str] = &[ "name", "title", "group", "id", "chno", "logo", @@ -809,7 +810,8 @@ pub(crate) struct Config { pub t_sources_file_path: String, #[serde(skip_serializing, skip_deserializing)] pub t_api_proxy_file_path: String, - + #[serde(skip)] + pub file_locks : Arc } impl Config { diff --git a/src/model/playlist.rs b/src/model/playlist.rs index 87a67f096..692841cf4 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -73,7 +73,7 @@ impl TryFrom for XtreamCluster { PlaylistItemType::Live => Ok(XtreamCluster::Live), PlaylistItemType::Video => Ok(XtreamCluster::Video), PlaylistItemType::Series => Ok(XtreamCluster::Series), - PlaylistItemType::SeriesInfo => Err(format!("Cant convert {item_type}")), + _ => Err(format!("Cant convert {item_type}")), } } } @@ -83,8 +83,10 @@ impl TryFrom for XtreamCluster { pub(crate) enum PlaylistItemType { Live = 1, Video = 2, - Series = 3, - SeriesInfo = 4, + Series = 3, // xtream series description + SeriesInfo = 4, // xtream series info fetched for series description + SeriesEpisode = 5, // from SeriesInfo parsed episodes + Catchup = 6, } impl Default for PlaylistItemType { @@ -110,6 +112,8 @@ impl Display for PlaylistItemType { PlaylistItemType::Video => "video", PlaylistItemType::Series => "series", PlaylistItemType::SeriesInfo => "series-info", + PlaylistItemType::SeriesEpisode => "series-episode", + PlaylistItemType::Catchup => "catchup" }) } } diff --git a/src/processing/playlist_processor.rs b/src/processing/playlist_processor.rs index c6d9b4d2b..7244f7ce1 100644 --- a/src/processing/playlist_processor.rs +++ b/src/processing/playlist_processor.rs @@ -466,7 +466,8 @@ fn flatten_groups(mut playlistgroups: Vec) -> Vec } async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>], - target: &ConfigTarget, cfg: &Config, + target: &ConfigTarget, + cfg: &Config, stats: &mut HashMap, errors: &mut Vec) -> Result<(), Vec> { let pipe = get_processing_pipe(target); diff --git a/src/repository/bplustree.rs b/src/repository/bplustree.rs index 2dc7a1bf7..5ed74b1a0 100644 --- a/src/repository/bplustree.rs +++ b/src/repository/bplustree.rs @@ -3,9 +3,9 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::marker::PhantomData; use std::path::Path; + use flate2::Compression; use log::error; - use serde::{Deserialize, Serialize}; const BINCODE_OVERHEAD: usize = 4; @@ -17,6 +17,18 @@ fn is_multiple_of_block_size(file: &File) -> io::Result { Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE } +fn is_file_valid(file: File) -> io::Result { + match is_multiple_of_block_size(&file) { + Ok(valid) => { + if !valid { + return Err(io::Error::new(io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); + } + } + Err(err) => return Err(err) + } + Ok(file) +} + #[inline] fn u32_from_bytes(bytes: &[u8]) -> io::Result { Ok(u32::from_le_bytes(bytes.try_into().map_err(|e: TryFromSliceError| io::Error::new(io::ErrorKind::Other, e.to_string()))?)) @@ -39,6 +51,25 @@ where } +fn get_entry_index_upper_bound(keys: &Vec, key: &K) -> usize +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut left = 0; + let mut right = keys.len(); + while left < right { + let mid = left + ((right - left) >> 1); + if &keys[mid] <= key { + left = mid + 1; + } else { + right = mid; + } + } + left +} + + #[derive(Serialize, Deserialize, Debug, Clone)] struct BPlusTreeNode { keys: Vec, @@ -81,6 +112,7 @@ where } } + #[allow(dead_code)] fn query(&self, key: &K) -> Option<&V> { if self.is_leaf { return match self.keys.binary_search(&key) { @@ -113,17 +145,7 @@ where } fn get_entry_index_upper_bound(&self, key: &K) -> usize { - let mut left = 0; - let mut right = self.keys.len(); - while left < right { - let mid = left + ((right - left) >> 1); - if &self.keys[mid] <= key { - left = mid + 1; - } else { - right = mid; - } - } - left + get_entry_index_upper_bound::(&self.keys, key) } fn insert(&mut self, key: K, v: V, inner_order: usize, leaf_order: usize) -> Option> { @@ -181,7 +203,7 @@ where self.children.iter().for_each(|child| child.traverse(visit)); } - fn serialize_to_blocks(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { + fn serialize_to_block(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { let mut current_offset = offset; let buffer_slice = &mut buffer[..]; @@ -204,7 +226,7 @@ where encoder.write_all(&values_encoded)?; let compressed_bytes = encoder.finish()?; let values_bytes_len = compressed_bytes.len(); - buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(values_bytes_len as u32).to_le_bytes()); + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(values_bytes_len as u32).to_le_bytes()); write_pos += 4; buffer_slice[write_pos..write_pos + values_bytes_len].copy_from_slice(&compressed_bytes); write_pos += values_bytes_len; @@ -220,7 +242,7 @@ where let mut pointer = Vec::with_capacity(self.children.len()); for child in &self.children { pointer.push(current_offset); - current_offset = child.serialize_to_blocks(file, buffer, current_offset)?; + current_offset = child.serialize_to_block(file, buffer, current_offset)?; } let pointer_encoded = bincode_serialize(&pointer)?; @@ -234,7 +256,7 @@ where Ok(current_offset) } - fn deserialize_from_blocks(file: &mut R, buffer: &mut Vec, offset: u64, nested: bool) -> io::Result<(Self, Option>)> { + fn deserialize_from_block(file: &mut R, buffer: &mut Vec, offset: u64, nested: bool) -> io::Result<(Self, Option>)> { file.seek(SeekFrom::Start(offset))?; file.read_exact(buffer)?; @@ -272,7 +294,7 @@ where let nodes: Result>, io::Error> = pointers .iter() .map(|pointer| { - BPlusTreeNode::::deserialize_from_blocks(file, buffer, *pointer, nested) + BPlusTreeNode::::deserialize_from_block(file, buffer, *pointer, nested) .map(|(node, _)| node) .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string())) }) @@ -352,6 +374,7 @@ where } } + #[allow(dead_code)] pub(crate) fn query(&self, key: &K) -> Option<&V> { self.root.query(key) } @@ -359,7 +382,7 @@ where pub(crate) fn serialize(&self, filepath: &Path) -> io::Result { let mut file = OpenOptions::new().write(true).create(true).open(filepath)?; let mut buffer = vec![0u8; BLOCK_SIZE]; - let result = self.root.serialize_to_blocks(&mut file, &mut buffer, 0u64); + let result = self.root.serialize_to_block(&mut file, &mut buffer, 0u64); file.flush()?; result } @@ -370,7 +393,7 @@ where return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); } let mut buffer = vec![0u8; BLOCK_SIZE]; - let (root, _) = BPlusTreeNode::deserialize_from_blocks(&mut file, &mut buffer, 0, true)?; + let (root, _) = BPlusTreeNode::deserialize_from_block(&mut file, &mut buffer, 0, true)?; Ok(BPlusTree::new_with_root(root)) } @@ -382,6 +405,33 @@ where } } +fn query_tree(file: &mut File, key: &K) -> Option +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut offset = 0; + let mut buffer = vec![0u8; BLOCK_SIZE]; + loop { + match BPlusTreeNode::::deserialize_from_block(file, &mut buffer, offset, false) { + Ok((node, pointers)) => { + if node.is_leaf { + return match node.keys.binary_search(key) { + Ok(idx) => node.values.get(idx).cloned(), + Err(_) => None, + }; + } + let child_idx = get_entry_index_upper_bound::(&node.keys, key); + offset = *pointers.unwrap().get(child_idx).unwrap(); + } + Err(err) => { + error!("Failed to read id tree from file {err}"); + return None; + } + }; + } +} + pub(crate) struct BPlusTreeQuery { file: File, _marker_k: PhantomData, @@ -394,15 +444,7 @@ where V: Serialize + for<'de> Deserialize<'de> + Clone, { pub(crate) fn try_new(filepath: &Path) -> io::Result { - let file = File::open(filepath)?; - match is_multiple_of_block_size(&file) { - Ok(valid) => { - if !valid { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); - } - } - Err(err) => return Err(err) - } + let file = is_file_valid(File::open(filepath)?)?; Ok(BPlusTreeQuery { file, _marker_k: Default::default(), @@ -410,44 +452,81 @@ where }) } - fn get_entry_index_upper_bound(keys: &Vec, key: &K) -> usize { - let mut left = 0; - let mut right = keys.len(); - while left < right { - let mid = left + ((right - left) >> 1); - if &keys[mid] <= key { - left = mid + 1; - } else { - right = mid; - } + pub(crate) fn query(&mut self, key: &K) -> Option { + query_tree(&mut self.file, key) + } +} + +pub(crate) struct BPlusTreeUpdate { + file: File, + _marker_k: PhantomData, + _marker_v: PhantomData, +} + +impl BPlusTreeUpdate +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn try_new(filepath: &Path) -> io::Result { + if !filepath.exists() { + return Err(io::Error::new(io::ErrorKind::NotFound, format!("File not found {}", filepath.to_str().unwrap_or("?")))); } - left + let file = OpenOptions::new() + .write(true) + .read(true) + .open(filepath)?; + let file = is_file_valid(file)?; + Ok(BPlusTreeUpdate { + file, + _marker_k: Default::default(), + _marker_v: Default::default(), + }) } pub(crate) fn query(&mut self, key: &K) -> Option { + query_tree(&mut self.file, key) + } + + fn serialize_node(&mut self, offset: u64, node: BPlusTreeNode) -> io::Result { + let mut buffer = vec![0u8; BLOCK_SIZE]; + let result = node.serialize_to_block(&mut self.file, &mut buffer, offset); + self.file.flush()?; + result + } + + pub(crate) fn update(&mut self, key: &K, value: V) -> io::Result { let mut offset = 0; let mut buffer = vec![0u8; BLOCK_SIZE]; loop { - match BPlusTreeNode::::deserialize_from_blocks(&mut self.file, &mut buffer, offset, false) { - Ok((node, pointers)) => { + match BPlusTreeNode::::deserialize_from_block(&mut self.file, &mut buffer, offset, false) { + Ok((mut node, pointers)) => { if node.is_leaf { return match node.keys.binary_search(key) { - Ok(idx) => node.values.get(idx).cloned(), - Err(_) => None, + Ok(idx) => { + let old_value = node.values.get(idx); + if let Some(_) = old_value { + node.values[idx] = value; + return self.serialize_node(offset, node); + } + Err(io::Error::new(io::ErrorKind::NotFound, "Entry not found")) + } + Err(_) => Err(io::Error::new(io::ErrorKind::NotFound, "Entry not found")), }; } - let child_idx = BPlusTreeQuery::::get_entry_index_upper_bound(&node.keys, key); + let child_idx = get_entry_index_upper_bound::(&node.keys, key); offset = *pointers.unwrap().get(child_idx).unwrap(); } Err(err) => { error!("Failed to read id tree from file {err}"); - return None; + return Err(io::Error::new(io::ErrorKind::NotFound, format!("Failed to read id tree from file {err}"))); } }; } } } + #[cfg(test)] mod tests { use std::io; @@ -455,7 +534,7 @@ mod tests { use serde::{Deserialize, Serialize}; - use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery}; + use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate}; // Example usage with a simple struct #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -507,6 +586,31 @@ mod tests { }), "Entry {} not found", i); } + let mut tree_update: BPlusTreeUpdate = BPlusTreeUpdate::try_new(&filepath)?; + for i in 0u32..=500 { + if let Some(record) = tree_update.query(&i) { + let new_record = Record { + id: record.id, + data: format!("Entry {}", record.id + 9000), + }; + tree_update.update(&i, new_record)?; + } else { + assert!(false, "Entry {} not found", i); + } + } + + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::try_new(&filepath)?; + for i in 0u32..=500 { + let found = tree_query.query(&i); + assert!(found.is_some(), "Entry {} not found", i); + let entry = found.unwrap(); + let expected = Record { + id: i, + data: format!("Entry {}", i + 9000), + }; + assert!(entry.eq(&expected), "Entry not equal {:?} != {:?}", entry, expected); + } + Ok(()) } } diff --git a/src/repository/id_mapping.rs b/src/repository/id_mapping.rs index 7ad77478a..a64199b0a 100644 --- a/src/repository/id_mapping.rs +++ b/src/repository/id_mapping.rs @@ -1,3 +1,5 @@ +// This module is not included in the build + use std::cmp::max; use std::io::Error; use std::path::{Path, PathBuf}; diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 5e1467612..b6028de9d 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -4,15 +4,15 @@ use std::path::{Path, PathBuf}; use log::error; -use crate::{create_m3u_filter_error}; use crate::api::api_utils::get_user_server_info; +use crate::create_m3u_filter_error; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{M3uPlaylistItem, PlaylistGroup, PlaylistItem, PlaylistItemType}; -use crate::repository::indexed_document_reader::{IndexedDocumentReader}; +use crate::repository::indexed_document_reader::IndexedDocumentReader; use crate::repository::indexed_document_writer::IndexedDocumentWriter; -use crate::repository::storage::{ensure_target_storage_path}; +use crate::repository::storage::ensure_target_storage_path; use crate::utils::file_utils; macro_rules! cant_write_result { @@ -64,25 +64,26 @@ fn persist_m3u_playlist_as_text(target: &ConfigTarget, cfg: &Config, m3u_playlis pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, target_path: &Path, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { if !new_playlist.is_empty() { if let Some((m3u_path, idx_path)) = m3u_get_file_paths(target_path) { - let m3u_playlist = new_playlist.iter() .flat_map(|pg| &pg.channels) .filter(|&pli| pli.header.borrow().item_type != PlaylistItemType::SeriesInfo) .map(PlaylistItem::to_m3u).collect::>(); persist_m3u_playlist_as_text(target, cfg, &m3u_playlist); - - match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) { - Ok(mut writer) => { - for m3u in m3u_playlist { - match writer.write_doc(m3u.virtual_id, &m3u) { - Ok(_) => {}, - Err(err) => return Err(cant_write_result!(&m3u_path, err)) + { + let _file_lock = cfg.file_locks.write_lock(&m3u_path).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, format!("{}", err)))?; + match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) { + Ok(mut writer) => { + for m3u in m3u_playlist { + match writer.write_doc(m3u.virtual_id, &m3u) { + Ok(_) => {} + Err(err) => return Err(cant_write_result!(&m3u_path, err)) + } } + writer.flush().map_err(|err| cant_write_result!(&m3u_path, err))?; } - writer.flush().map_err(|err| cant_write_result!(&m3u_path, err))?; + Err(err) => return Err(cant_write_result!(&m3u_path, err)) } - Err(err) => return Err(cant_write_result!(&m3u_path, err)) } } } @@ -93,36 +94,43 @@ pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, use match ensure_target_storage_path(cfg, target.name.as_str()) { Ok(target_path) => { if let Some((m3u_path, _)) = m3u_get_file_paths(&target_path) { - match IndexedDocumentReader::::new(&m3u_path) { - Ok(mut reader) => { - let server_info = get_user_server_info(cfg, user); - let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); - let mut result = vec![]; - result.push("#EXTM3U".to_string()); - for m3u_pli in reader.by_ref() { - match user.proxy { - ProxyType::Reverse => { - result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{}", m3u_pli.virtual_id).as_str()))); - } - ProxyType::Redirect => { - result.push(m3u_pli.to_m3u(target, None)); + { + let _file_lock = cfg.file_locks.read_lock(&m3u_path).map_err(|err| { + error!("Could not lock document {:?}: {}", m3u_path, err); + Error::new(ErrorKind::Other, format!("Document Reader error for target {}", &target.name)) + }).ok()?; + + match IndexedDocumentReader::::new(&m3u_path) { + Ok(mut reader) => { + let server_info = get_user_server_info(cfg, user); + let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); + let mut result = vec![]; + result.push("#EXTM3U".to_string()); + for m3u_pli in reader.by_ref() { + match user.proxy { + ProxyType::Reverse => { + result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{}", m3u_pli.virtual_id).as_str()))); + } + ProxyType::Redirect => { + result.push(m3u_pli.to_m3u(target, None)); + } } + }; + if reader.by_ref().has_error() { + error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap()); + } else { + return Some(result.join("\n")); } - }; - if reader.by_ref().has_error() { - error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap()); - } else { - return Some(result.join("\n")); } - } - Err(err) => { - error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err); + Err(err) => { + error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err); + } } } } else { error!("Could not open files for target {}", &target.name); } - }, + } Err(err) => { error!("Could not find storage path for target {} - {}", target.name.as_str(), err); } @@ -130,9 +138,12 @@ pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, use None } -pub(crate) fn m3u_get_item_for_stream_id(stream_id: u32, m3u_path: &Path, idx_path: &Path) -> Result { +pub(crate) fn m3u_get_item_for_stream_id(cfg: &Config, stream_id: u32, m3u_path: &Path, idx_path: &Path) -> Result { if stream_id < 1 { return Err(Error::new(ErrorKind::Other, "id should start with 1")); } - IndexedDocumentReader::::read_indexed_item(m3u_path, idx_path, stream_id) + { + let _file_lock = cfg.file_locks.read_lock(&m3u_path)?; + IndexedDocumentReader::::read_indexed_item(m3u_path, idx_path, stream_id) + } } \ No newline at end of file diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 91382caaf..7db3f6877 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -8,5 +8,4 @@ pub(crate) mod storage; mod indexed_document_writer; mod indexed_document_reader; pub(crate) mod target_id_mapping; -pub(crate) mod id_mapping; pub(crate) mod bplustree; diff --git a/src/repository/playlist_repository.rs b/src/repository/playlist_repository.rs index 38767d3b2..298369eed 100644 --- a/src/repository/playlist_repository.rs +++ b/src/repository/playlist_repository.rs @@ -15,54 +15,61 @@ pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg> match ensure_target_storage_path(cfg, target.name.as_str()) { Ok(target_path) => { - // reassign old virtual id or assign a new one - let mut target_id_mapping = TargetIdMapping::new(&get_target_id_mapping_file(&target_path)); - for group in &mut *playlist { - for channel in &group.channels { - let mut header = channel.header.borrow_mut(); - match header.get_provider_id() { - Some(provider_id) => { - let uuid = header.get_uuid(); - let item_type = header.item_type; - header.virtual_id = match target_id_mapping.get_by_uuid(&header.uuid) { - None => target_id_mapping.insert_entry(provider_id, **uuid, &item_type, 0), - Some(existing_id) => *existing_id - }; + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + match cfg.file_locks.write_lock(&target_id_mapping_file) { + Ok(_file_lock) => { + // reassign old virtual id or assign a new one + let mut target_id_mapping = TargetIdMapping::new(&target_id_mapping_file); + for group in &mut *playlist { + for channel in &group.channels { + let mut header = channel.header.borrow_mut(); + match header.get_provider_id() { + Some(provider_id) => { + let uuid = header.get_uuid(); + let item_type = header.item_type; + header.virtual_id = target_id_mapping.insert_entry(**uuid, provider_id, &item_type, 0); + } + None => { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Playlistitem has no provider id: {}", &header.title))); + } + } + } } - None => { - errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Playlistitem has no provider id: {}", &header.title))); - } - } - } - } - for output in &target.output { - match match output.target { - TargetType::M3u => m3u_write_playlist(target, cfg, &target_path, playlist), - TargetType::Xtream => xtream_write_playlist(target, cfg, playlist), - TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), - } { - Ok(()) => { - if let Err(err) = target_id_mapping.persist() { - errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); - } - if !playlist.is_empty() { - match epg_write(target, cfg, &target_path, epg, output) { - Ok(()) => {} + for output in &target.output { + match match output.target { + TargetType::M3u => m3u_write_playlist(target, cfg, &target_path, playlist), + TargetType::Xtream => xtream_write_playlist(target, cfg, playlist), + TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), + } { + Ok(()) => { + if let Err(err) = target_id_mapping.persist() { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } + if !playlist.is_empty() { + match epg_write(target, cfg, &target_path, epg, output) { + Ok(()) => {} + Err(err) => errors.push(err) + } + } + } Err(err) => errors.push(err) } } + if let Err(err) = target_id_mapping.persist() { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } + } + Err(err) => { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); } - Err(err) => errors.push(err) } } - if let Err(err) = target_id_mapping.persist() { - errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); - } } Err(err) => { errors.push(err); - }, + } }; if errors.is_empty() { Ok(()) } else { Err(errors) } diff --git a/src/repository/storage.rs b/src/repository/storage.rs index 62e899703..a2ed476b9 100644 --- a/src/repository/storage.rs +++ b/src/repository/storage.rs @@ -8,7 +8,7 @@ pub(crate) fn hash_string(url: &str) -> [u8; 32] { hash.into() // Konvertiere den Hash in ein Array mit fester Größe } -pub(crate) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { +pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { target_path.join(PathBuf::from("id_mapping.db")) } diff --git a/src/repository/target_id_mapping.rs b/src/repository/target_id_mapping.rs index 2cbdecbbe..5e204c124 100644 --- a/src/repository/target_id_mapping.rs +++ b/src/repository/target_id_mapping.rs @@ -31,6 +31,10 @@ impl VirtualIdRecord { pub(crate) fn is_expired(&self) -> bool { (Local::now().timestamp() - self.last_updated) > EXPIRATION_DURATION } + + pub(crate) fn copy_update_timestamp(&self) -> Self { + Self::new(self.provider_id, self.virtual_id, self.item_type, self.parent_virtual_id, self.uuid) + } } pub(crate) struct TargetIdMapping { @@ -69,16 +73,17 @@ impl TargetIdMapping { } } - pub(crate) fn insert_entry(&mut self, provider_id: u32, uuid: [u8; 32], item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { - self.dirty = true; - self.virtual_id_counter += 1; - let record = VirtualIdRecord::new(provider_id, self.virtual_id_counter, *item_type, parent_virtual_id, uuid); - self.by_virtual_id.insert(self.virtual_id_counter, record); - self.virtual_id_counter - } - - pub(crate) fn get_by_uuid(&self, uuid: &[u8; 32]) -> Option<&u32> { - self.by_uuid.get(uuid) + pub(crate) fn insert_entry(&mut self, uuid: [u8; 32], provider_id: u32, item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { + match self.by_uuid.get(&uuid) { + None => { + self.dirty = true; + self.virtual_id_counter += 1; + let record = VirtualIdRecord::new(provider_id, self.virtual_id_counter, *item_type, parent_virtual_id, uuid); + self.by_virtual_id.insert(self.virtual_id_counter, record); + self.virtual_id_counter + } + Some(record) => *record + } } pub(crate) fn persist(&mut self) -> Result<(), Error> { diff --git a/src/repository/xtream_repository.rs b/src/repository/xtream_repository.rs index f03b97e34..bd0327aed 100644 --- a/src/repository/xtream_repository.rs +++ b/src/repository/xtream_repository.rs @@ -11,9 +11,8 @@ use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; -use crate::repository::bplustree::{BPlusTreeQuery}; -use crate::repository::id_mapping::IdMapping; -use crate::repository::indexed_document_reader::{IndexedDocumentReader}; +use crate::repository::bplustree::{BPlusTreeQuery, BPlusTreeUpdate}; +use crate::repository::indexed_document_reader::IndexedDocumentReader; use crate::repository::indexed_document_writer::IndexedDocumentWriter; use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path, hash_string}; use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord}; @@ -56,29 +55,28 @@ fn xtream_get_info_file_paths(storage_path: &Path, cluster: XtreamCluster) -> Op None } -fn xtream_get_catchup_id_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("catchup_mapping.db") -} - -fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { +fn write_playlists_to_file(cfg: &Config, storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { for (cluster, playlist) in collections { let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, &cluster); - match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { - Ok(mut writer) => { - for item in playlist { - match item.to_xtream() { - Ok(xtream) => { - match writer.write_doc(item.header.borrow().virtual_id, &xtream) { - Ok(_) => {} - Err(err) => return Err(cant_write_result!(&xtream_path, err)) + { + let _file_lock = cfg.file_locks.write_lock(&xtream_path).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, format!("{}", err)))?; + match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { + Ok(mut writer) => { + for item in playlist { + match item.to_xtream() { + Ok(xtream) => { + match writer.write_doc(item.header.borrow().virtual_id, &xtream) { + Ok(_) => {} + Err(err) => return Err(cant_write_result!(&xtream_path, err)) + } } - }, - Err(err) => return Err(cant_write_result!(&xtream_path, err)) + Err(err) => return Err(cant_write_result!(&xtream_path, err)) + } } + writer.flush().map_err(|err| cant_write_result!(&xtream_path, err))?; } - writer.flush().map_err(|err| cant_write_result!(&xtream_path, err))?; + Err(err) => return Err(cant_write_result!(&xtream_path, err)) } - Err(err) => return Err(cant_write_result!(&xtream_path, err)) } } Ok(()) @@ -221,7 +219,7 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis } } - match write_playlists_to_file(&path, vec![ + match write_playlists_to_file(cfg, &path, vec![ (XtreamCluster::Live, &mut live_col), (XtreamCluster::Video, &mut vod_col), (XtreamCluster::Series, &mut series_col)]) { @@ -250,186 +248,222 @@ pub(crate) fn xtream_get_collection_path(cfg: &Config, target_name: &str, collec Err(Error::new(ErrorKind::Other, format!("Cant find collection: {target_name}/{collection_name}"))) } -fn xtream_read_item_for_stream_id(stream_id: u32, storage_path: &Path, cluster: &XtreamCluster) -> Result { +fn xtream_read_item_for_stream_id(cfg: &Config, stream_id: u32, storage_path: &Path, cluster: &XtreamCluster) -> Result { let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); - return IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id); -} - -fn xtream_read_series_item_for_stream_id(stream_id: u32, storage_path: &Path) -> Result { - let (xtream_path, idx_path) = xtream_get_file_paths_for_series(storage_path); - return IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id); -} - -pub(crate) fn xtream_get_item_for_stream_id(virtual_id: u32, config: &Config, target: &ConfigTarget, xtream_cluster: Option) -> Result { - if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { - if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { - match BPlusTreeQuery::::try_new(&get_target_id_mapping_file(&target_path)) { - Ok(mut target_id_mapping) => { - return match target_id_mapping.query(&virtual_id) { - Some(mapping) => { - if mapping.item_type == PlaylistItemType::SeriesInfo { - xtream_read_series_item_for_stream_id(virtual_id, &storage_path) - } else if mapping.item_type == PlaylistItemType::Series && mapping.parent_virtual_id > 0 { - // we load the original series item - match xtream_read_series_item_for_stream_id(mapping.parent_virtual_id, &storage_path) { - Ok(mut item) => { - // we need to replace the provider id with the episode provider id - item.provider_id = mapping.provider_id; - Ok(item) - }, - Err(err) => Err(err) - } - } else { - let cluster = match xtream_cluster { - Some(c) => Some(c), - None => match XtreamCluster::try_from(mapping.item_type) { - Ok(item_type) => Some(item_type), - Err(_) => None - } - }; - match cluster { - Some(xc) => xtream_read_item_for_stream_id(virtual_id, &storage_path, &xc), - None => Err(Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {virtual_id}"))) - } - } - }, - None => Err(Error::new(ErrorKind::Other, format!("Could not find mappping for target {} and id {}", target.name, virtual_id))), - }; - }, - Err(err) => return Err(Error::new(ErrorKind::Other, format!("Could not load id mappping for target {} err:{}", target.name, err.to_string()))) - }; - } else { - return Err(Error::new(ErrorKind::Other, format!("Could not find path for target {} xtream output", &target.name))); - } - } else { - return Err(Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name))); + { + let _file_lock = cfg.file_locks.read_lock(&xtream_path)?; + IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id) } } +fn xtream_read_series_item_for_stream_id(cfg: &Config, stream_id: u32, storage_path: &Path) -> Result { + let (xtream_path, idx_path) = xtream_get_file_paths_for_series(storage_path); + { + let _file_lock = cfg.file_locks.read_lock(&xtream_path)?; + return IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id); + } +} + +macro_rules! try_cluster { + ($xtream_cluster:expr, $item_type:expr, $virtual_id:expr) => { + $xtream_cluster.or_else(|| XtreamCluster::try_from($item_type).ok()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {}", $virtual_id))) + }; +} + +pub(crate) fn xtream_get_item_for_stream_id( + virtual_id: u32, + config: &Config, + target: &ConfigTarget, + xtream_cluster: Option, +) -> Result { + let target_path = get_target_storage_path(config, target.name.as_str()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name)))?; + let storage_path = xtream_get_storage_path(config, target.name.as_str()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {} xtream output", &target.name)))?; + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.read_lock(&target_id_mapping_file) + .map_err(|err| Error::new(ErrorKind::Other, format!("Could not get lock for id mapping for target {} err:{}", target.name, err.to_string())))?; + + let mut target_id_mapping = BPlusTreeQuery::::try_new(&target_id_mapping_file) + .map_err(|err| Error::new(ErrorKind::Other, format!("Could not load id mapping for target {} err:{}", target.name, err.to_string())))?; + + let mapping = target_id_mapping + .query(&virtual_id) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find mapping for target {} and id {}", target.name, virtual_id)))?; + + match mapping.item_type { + PlaylistItemType::SeriesInfo => xtream_read_series_item_for_stream_id(config, virtual_id, &storage_path), + PlaylistItemType::SeriesEpisode => { + let mut item = xtream_read_series_item_for_stream_id(config, mapping.parent_virtual_id, &storage_path)?; + item.provider_id = mapping.provider_id; + Ok(item) + } + PlaylistItemType::Catchup => { + let cluster = try_cluster!(xtream_cluster, mapping.item_type, virtual_id)?; + let mut item = xtream_read_item_for_stream_id(config, mapping.parent_virtual_id, &storage_path, &cluster)?; + item.provider_id = mapping.provider_id; + Ok(item) + } + _ => { + let cluster = try_cluster!(xtream_cluster, mapping.item_type, virtual_id)?; + xtream_read_item_for_stream_id(config, virtual_id, &storage_path, &cluster) + } + } + } +} + + pub(crate) fn xtream_load_rewrite_playlist(cluster: &XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result { if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { let (xtream_path, _) = xtream_get_file_paths(&storage_path, cluster); - match IndexedDocumentReader::::new(&xtream_path) { - Ok(mut reader) => { - let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); - let result: Vec = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id) - .map(|pli| pli.to_doc(&options)).collect(); - if reader.by_ref().has_error() { - error!("Could not deserialize item {}", &xtream_path.to_str().unwrap()); - } else { - return Ok(serde_json::to_string(&result).unwrap()); + { + let _file_lock = config.file_locks.read_lock(&xtream_path)?; + match IndexedDocumentReader::::new(&xtream_path) { + Ok(mut reader) => { + let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); + let result: Vec = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id) + .map(|pli| pli.to_doc(&options)).collect(); + if reader.by_ref().has_error() { + error!("Could not deserialize item {}", &xtream_path.to_str().unwrap()); + } else { + return Ok(serde_json::to_string(&result).unwrap()); + } + } + Err(err) => { + error!("Could not deserialize file {} - {}", &xtream_path.to_str().unwrap(), err); } - } - Err(err) => { - error!("Could not deserialize file {} - {}", &xtream_path.to_str().unwrap(), err); } } } Err(Error::new(ErrorKind::Other, format!("Failed to find xtream storage for target {}", &target.name))) } +macro_rules! try_option_ok { + ($option:expr) => { + match $option { + Some(value) => value, + None => return Ok(()), + } + }; +} pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, - series_id: u32, + series_info_id: u32, content: &str) -> Result<(), Error> { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { - return match IndexedDocumentWriter::new_append(info_path.clone(), idx_path) { - Ok(mut writer) => { - match writer.write_doc(series_id, content) { - Ok(_) => {}, - Err(_) => return Err(Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}"))) - } - return Ok(writer.flush()?); - } - Err(err) => Err(err) - }; - } + let target_path = try_option_ok!(get_target_storage_path(config, target_name)); + let storage_path = try_option_ok!(xtream_get_storage_path(config, target_name)); + let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)); + + { + let _file_lock = config.file_locks.write_lock(&info_path)?; + let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?; + + writer + .write_doc(series_info_id, content) + .map_err(|_| Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}")))?; + + writer.flush()?; } + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.write_lock(&target_id_mapping_file)?; + if let Ok(mut target_id_mapping) = BPlusTreeUpdate::::try_new(&target_id_mapping_file) { + if let Some(record) = target_id_mapping.query(&series_info_id) { + let new_record = record.copy_update_timestamp(); + let _ = target_id_mapping.update(&series_info_id, new_record); + } + }; + } + Ok(()) } // Reads the series info entry if exists, otherwise error -pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { - if info_path.exists() && idx_path.exists() { - return IndexedDocumentReader::::read_indexed_item(&info_path, &idx_path, series_id); +pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Option { + let target_path = get_target_storage_path(config, target_name)?; + let storage_path = xtream_get_storage_path(config, target_name)?; + + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.read_lock(&target_id_mapping_file).map_err(|err| { + error!("Could not lock id mapping for target {target_name}: {}", err); + Error::new(ErrorKind::Other, format!("ID mapping load error for target {target_name}")) + }).ok()?; + let mut target_id_mapping = BPlusTreeQuery::::try_new(&target_id_mapping_file) + .map_err(|err| { + error!("Could not load id mapping for target {target_name}: {}", err); + Error::new(ErrorKind::Other, format!("ID mapping load error for target {target_name}")) + }).ok()?; + + if let Some(id_record) = target_id_mapping.query(&series_id) { + if id_record.is_expired() { + return None; } } } - Err(Error::new(ErrorKind::Other, format!("Failed to read series info for id {series_id} for {target_name}"))) -} -pub(crate) fn xtream_load_catchup_id_mapping(config: &Config, target_name: &str) -> Result, Error> { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let catchup_file = xtream_get_catchup_id_mapping_file_path(&storage_path); - return Ok(IdMapping::::new(&catchup_file)); - } - Err(Error::new(ErrorKind::Other, format!("Failed to load catchup id mapping {target_name}"))) -} + let (info_path, idx_path) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)?; -pub(crate) fn write_and_get_xtream_series_info(config: &Config, target: &ConfigTarget, pli_series_info: &XtreamPlaylistItem, content: &str) -> Result { - if let Ok(mut doc) = serde_json::from_str::(content) { - if let Some(target_path) = get_target_storage_path(config, target.name.as_str()) { - let mut target_id_mapping = TargetIdMapping::new(&get_target_id_mapping_file(&target_path)); - if let Some(episodes) = doc.get_mut("episodes").and_then(|e| e.as_object_mut()) { - let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); - for episode_list in episodes.values_mut() { - if let Some(entries) = episode_list.as_array_mut() { - for episode in entries.iter_mut().filter_map(|e| e.as_object_mut()) { - if let Some(episode_id) = episode.get("id").and_then(|id| id.as_str()) { - if let Ok(provider_id) = episode_id.parse::() { - let uuid = hash_string(&format!("{}/{}", pli_series_info.url, provider_id)); - let virtual_id = target_id_mapping.insert_entry(provider_id, uuid, &PlaylistItemType::Series, pli_series_info.virtual_id); - episode.insert("id".to_string(), Value::String(virtual_id.to_string())); - } - } - - if options.skip_series_direct_source { - episode.insert("direct_source".to_string(), Value::String(String::new())); - } - } - } + if info_path.exists() && idx_path.exists() { + { + let _file_lock = config.file_locks.read_lock(&info_path).map_err(|err| { + error!("Could not lock document {:?}: {}", info_path, err); + Error::new(ErrorKind::Other, format!("Document Reader error for target {target_name}")) + }).ok()?; + return match IndexedDocumentReader::::read_indexed_item(&info_path, &idx_path, series_id) { + Ok(content) => Some(content), + Err(err) => { + error!("Failed to read series info for id {series_id} for {target_name}: {}", err); + None } - - drop(target_id_mapping); - if let Ok(result) = serde_json::to_string(&doc) { - let _ = xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result); - return Ok(result); - } - } - - - // if let Some(episodes) = doc.get_mut("episodes") { - // if let Some(episodes_map) = episodes.as_object_mut() { - // let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); - // for (_season, episode_list) in episodes_map { - // // Iterate over items in the episode - // if let Some(entries) = episode_list.as_array_mut() { - // for entry in entries { - // if let Some(episode) = entry.as_object_mut() { - // if let Some(episode_id) = episode.get("id") { - // if let Ok(provider_id) = episode_id.as_str().unwrap().parse::() { - // let uuid = hash_string(format!("{}/{}", pli_series_info.url, provider_id).as_str()); - // let virtual_id = target_id_mapping.insert_entry(provider_id, uuid, &PlaylistItemType::Series, pli_series_info.virtual_id); - // episode.insert("id".to_string(), Value::String(virtual_id.to_string())); - // } - // } - // if options.skip_series_direct_source { - // episode.insert("direct_source".to_string(), Value::String(String::new())); - // } - // } - // } - // } - // } - // } - // drop(target_id_mapping); - // if let Ok(result) = serde_json::to_string(&doc) { - // let _ = xtream_repository::xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result); - // return Ok(result); - // } - // } + }; } } - Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli_series_info.virtual_id))) + None +} + +pub(crate) fn write_and_get_xtream_series_info( + config: &Config, + target: &ConfigTarget, + pli_series_info: &XtreamPlaylistItem, + content: &str, +) -> Result { + let mut doc = serde_json::from_str::(content) + .map_err(|_| Error::new(ErrorKind::Other, "Failed to parse JSON content"))?; + + let target_path = get_target_storage_path(config, target.name.as_str()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {}", target.name)))?; + + let episodes = doc.get_mut("episodes") + .and_then(Value::as_object_mut) + .ok_or_else(|| Error::new(ErrorKind::Other, "No episodes found in content"))?; + + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.write_lock(&target_id_mapping_file) + .map_err(|err| Error::new(ErrorKind::Other, format!("Could not load id mapping for target {} err:{}", target.name, err.to_string())))?; + let mut target_id_mapping = TargetIdMapping::new(&target_id_mapping_file); + let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); + + for episode_list in episodes.values_mut().filter_map(Value::as_array_mut) { + for episode in episode_list.iter_mut().filter_map(Value::as_object_mut) { + if let Some(provider_id) = episode.get("id").and_then(Value::as_str).and_then(|id| id.parse::().ok()) { + let uuid = hash_string(&format!("{}/{}", pli_series_info.url, provider_id)); + let virtual_id = target_id_mapping.insert_entry(uuid, provider_id, &PlaylistItemType::SeriesEpisode, pli_series_info.virtual_id); + episode.insert("id".to_string(), Value::String(virtual_id.to_string())); + } + if options.skip_series_direct_source { + episode.insert("direct_source".to_string(), Value::String(String::new())); + } + } + } + + drop(target_id_mapping); + } + let result = serde_json::to_string(&doc) + .map_err(|_| Error::new(ErrorKind::Other, "Failed to serialize updated series info"))?; + xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result).ok(); + + Ok(result) } \ No newline at end of file diff --git a/src/utils/file_lock_manager.rs b/src/utils/file_lock_manager.rs new file mode 100644 index 000000000..6e16675e3 --- /dev/null +++ b/src/utils/file_lock_manager.rs @@ -0,0 +1,106 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::{fmt, io}; +use std::path::{Path, PathBuf}; + +#[derive(Clone)] +pub(crate) struct FileLockManager { + locks: Arc>>>>, +} + +impl FileLockManager { + pub(crate) fn new() -> Self { + Self { + locks: Arc::new(Mutex::new(HashMap::new())), + } + } + + // Acquires a read lock for the specified file and returns a FileReadGuard. + pub(crate) fn read_lock(&self, path: &Path) -> io::Result { + let file_lock = self.get_or_create_lock(path)?; + let guard = file_lock.read().map_err(|_| { + io::Error::new(io::ErrorKind::Other, "Failed to acquire read lock") + })?; + // Clone the Arc to avoid moving `file_lock` out, as it is still borrowed by `guard` + Ok(FileReadGuard::new(Arc::clone(&file_lock), guard)) + } + + // Acquires a write lock for the specified file and returns a FileWriteGuard. + pub(crate) fn write_lock(&self, path: &Path) -> io::Result { + let file_lock = self.get_or_create_lock(path)?; + let guard = file_lock.write().map_err(|_| { + io::Error::new(io::ErrorKind::Other, "Failed to acquire write lock") + })?; + // Clone the Arc to avoid moving `file_lock` out, as it is still borrowed by `guard` + Ok(FileWriteGuard::new(Arc::clone(&file_lock), guard)) + } + + // Helper function: retrieves or creates a lock for a file. + fn get_or_create_lock(&self, path: &Path) -> io::Result>> { + let mut locks = self.locks.lock().map_err(|_| { + io::Error::new(io::ErrorKind::Other, "Failed to acquire lock on lock manager") + })?; + + if let Some(lock) = locks.get(path) { + return Ok(lock.clone()); + } + + let file_lock = Arc::new(RwLock::new(())); + locks.insert(path.to_path_buf(), file_lock.clone()); + Ok(file_lock) + } +} + +impl Default for FileLockManager { + fn default() -> Self { + FileLockManager::new() + } +} + +impl fmt::Debug for FileLockManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Acquire the lock to safely access the HashMap + let locks = self.locks.lock().unwrap(); + // Format the paths in the HashMap for debug purposes + let keys: Vec<_> = locks.keys().collect(); + f.debug_struct("FileLockManager") + .field("locks", &keys) + .finish() + } +} + +// Define FileReadGuard to hold both the lock reference and the actual read guard. +#[allow(dead_code)] +pub(crate) struct FileReadGuard { + lock: Arc>, + guard: RwLockReadGuard<'static, ()>, +} + +impl FileReadGuard { + pub fn new(lock: Arc>, guard: RwLockReadGuard<'_, ()>) -> Self { + // Convert the lifetime of `guard` to 'static by transmuting. + let static_guard: RwLockReadGuard<'static, ()> = unsafe { std::mem::transmute(guard) }; + FileReadGuard { + lock, + guard: static_guard, + } + } +} + +// Define FileWriteGuard to hold both the lock reference and the actual write guard. +#[allow(dead_code)] +pub(crate) struct FileWriteGuard { + lock: Arc>, + guard: RwLockWriteGuard<'static, ()>, +} + +impl FileWriteGuard { + pub fn new(lock: Arc>, guard: RwLockWriteGuard<'_, ()>) -> Self { + // Convert the lifetime of `guard` to 'static by transmuting. + let static_guard: RwLockWriteGuard<'static, ()> = unsafe { std::mem::transmute(guard) }; + FileWriteGuard { + lock, + guard: static_guard, + } + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 008bd750c..be648a6f0 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,4 +5,5 @@ pub (crate) mod string_utils; pub (crate) mod json_utils; pub (crate) mod config_reader; pub (crate) mod default_utils; -pub (crate) mod multi_file_reader; \ No newline at end of file +pub (crate) mod multi_file_reader; +pub (crate) mod file_lock_manager; \ No newline at end of file