diff --git a/CHANGELOG.md b/CHANGELOG.md index eca921869..0e38ffc5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +# 2.0.6 (2024-10-*) +- breaking change virtual_id handling. You need to clear the data directory. + # 2.0.5(2024-10-16) - input url supports now scheme `file://...` (which is not necessary because file paths are supported). Gzip files are also supported. - sort takes now a sequence for channel values which has higher priority than sort order diff --git a/Cargo.lock b/Cargo.lock index 5122f908d..c4f8a86e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,9 +506,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bytestring" @@ -527,9 +527,9 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" [[package]] name = "cc" -version = "1.1.30" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "jobserver", "libc", @@ -760,9 +760,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -1259,9 +1259,9 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e658178c10c747241199382079c0f195ce229866fbf4aa0d46fa6107fe33d2ec" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" [[package]] name = "indexmap" @@ -1370,9 +1370,9 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libnghttp2-sys" @@ -1445,7 +1445,6 @@ dependencies = [ "actix-server", "actix-web", "actix-web-httpauth", - "base64 0.22.1", "bincode", "blake3", "chrono", @@ -1607,9 +1606,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" -version = "0.10.67" +version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8cefcf97f41316955f9294cd61f639bdcfa9f2f230faac6cb896aa8ab64704" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -1639,9 +1638,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.3.2+3.3.2" +version = "300.4.0+3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b" +checksum = "a709e02f2b4aca747929cca5ed248880847c650233cf8b8cdc48f40aaf4898a6" dependencies = [ "cc", ] @@ -1785,18 +1784,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", @@ -1805,9 +1804,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -1854,9 +1853,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -2149,9 +2148,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "once_cell", "ring", @@ -2239,18 +2238,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", @@ -2259,9 +2258,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2405,9 +2404,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.79" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -2459,18 +2458,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", @@ -2525,9 +2524,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -2642,12 +2641,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" diff --git a/Cargo.toml b/Cargo.toml index ba849560a..8409d02a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ strip = true [dependencies] serde = { version = "1.0", features = ["derive", "rc"] } -serde_yaml = "0.9.33" +serde_yaml = "0.9.34" serde_json = "1" quick-xml = { version = "0.36", features = ["serialize"] } -regex = "1.10" +regex = "1.11" clap = { version = "4", features = ["derive"] } url = "2.5" reqwest = { version = "0", features = ["blocking", "json", "stream", "rustls-tls"] } @@ -49,5 +49,4 @@ rand = "0.8" rpassword = "7.3" flate2 = "1" time = "0.3" -blake3 = "1.5.4" -base64 = "0.22.1" +blake3 = "1.5" diff --git a/bin/print_idx_file.py b/bin/print_idx_file.py deleted file mode 100755 index 8afeff2d4..000000000 --- a/bin/print_idx_file.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -""" -This script reads the index files for xtream file indexes -Each index entry consist of 8 bytes where the first 4 bytes and the last 4 bytes are unsigned 32-bit integers (u32). -@see IndexRecord - -The IndexRecord can be use for different purposes, f.e. - - id -> id mapping - - id -> index - - index -> size (in this case the offset for the id is calculated through the "cluster_id_start->cluster mapping" offset shift) -""" - -import sys -import struct - -def print_u32_values(file_path): - try: - with open(file_path, "rb") as file: - while True: - chunk = file.read(8) - if not chunk: - break - - if len(chunk) == 8: - first_u32, last_u32 = struct.unpack("") - sys.exit(1) - - file_path = sys.argv[1] - print_u32_values(file_path) - diff --git a/src/api/m3u_api.rs b/src/api/m3u_api.rs index 94d72fd98..fab63837f 100644 --- a/src/api/m3u_api.rs +++ b/src/api/m3u_api.rs @@ -5,6 +5,7 @@ use crate::api::api_utils::{get_user_target, get_user_target_by_credentials, str use crate::api::api_model::{AppState, UserApiRequest}; use crate::model::config::TargetType; use crate::repository::m3u_repository::{m3u_get_file_paths, m3u_get_item_for_stream_id, m3u_load_rewrite_playlist}; +use crate::repository::storage::get_target_storage_path; async fn m3u_api( api_req: web::Query, @@ -15,7 +16,6 @@ async fn m3u_api( //let api_req = UserApiRequest::from_map(&_api_req); match get_user_target(&api_req, &app_state) { Some((user, target)) => { - // let filename = target.get_m3u_filename(); if let Some(content) = m3u_load_rewrite_playlist(&app_state.config, target, &user) { HttpResponse::Ok().content_type(mime::TEXT_PLAIN_UTF_8).body(content) } else { @@ -38,15 +38,22 @@ async fn m3u_api_stream( if let Ok(m3u_stream_id) = stream_id.parse::() { if let Some((_user, target)) = get_user_target_by_credentials(&username, &password, &api_req, &app_state) { if target.has_output(&TargetType::M3u) { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&app_state.config, target) { - match m3u_get_item_for_stream_id(m3u_stream_id, &m3u_path, &idx_path) { - Ok(m3u_item) => { - return stream_response(m3u_item.url.as_str(), &req, None).await; - } - Err(err) => { - error!("Failed to get m3u url: {}", err); + match get_target_storage_path(&app_state.config, target.name.as_str()) { + Some(target_path) => { + if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) { + match m3u_get_item_for_stream_id(&app_state.config, m3u_stream_id, &m3u_path, &idx_path) { + Ok(m3u_item) => { + return stream_response(m3u_item.url.as_str(), &req, None).await; + } + Err(err) => { + error!("Failed to get m3u url: {}", err); + } + } } } + None => { + error!("Failed to get target path for {}", target.name); + } } } } diff --git a/src/api/mod.rs b/src/api/mod.rs index e18209461..9f97f7f37 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,4 +7,4 @@ mod xtream_api; mod m3u_api; mod xmltv_api; mod scheduler; -mod web_index; +mod web_index; \ No newline at end of file diff --git a/src/api/scheduler.rs b/src/api/scheduler.rs index 4c0d0c6e1..a7286c739 100644 --- a/src/api/scheduler.rs +++ b/src/api/scheduler.rs @@ -1,29 +1,81 @@ use std::str::FromStr; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use actix_web::web::Data; -use chrono::Local; +use chrono::{DateTime, FixedOffset, Local}; use cron::Schedule; use log::error; use crate::api::api_model::AppState; use crate::exit; use crate::processing::playlist_processor::exec_processing; +fn datetime_to_instant(datetime: DateTime) -> Instant { + // Convert DateTime to SystemTime + let target_system_time: SystemTime = datetime.into(); + + // Get the current SystemTime + let now_system_time = SystemTime::now(); + + // Calculate the duration between now and the target time + let duration_until = target_system_time + .duration_since(now_system_time) + .unwrap_or_else(|_| Duration::from_secs(0)); + + // Get the current Instant and add the duration to calculate the target Instant + Instant::now() + duration_until +} + pub(crate) async fn start_scheduler(expression: &str, data: Data) -> ! { match Schedule::from_str(expression) { Ok(schedule) => { let offset = *Local::now().offset(); loop { let mut upcoming = schedule.upcoming(offset).take(1); - actix_rt::time::sleep(Duration::from_millis(500)).await; - let local = &Local::now(); - if let Some(datetime) = upcoming.next() { - if datetime.timestamp() <= local.timestamp() { - exec_processing(data.config.clone(), data.targets.clone()).await; - } + actix_rt::time::sleep_until(actix_rt::time::Instant::from(datetime_to_instant(datetime))).await; + exec_processing(data.config.clone(), data.targets.clone()).await; } } } Err(err) => exit!("Failed to start scheduler: {}", err) } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use std::sync::atomic::{AtomicU8, Ordering}; + use chrono::Local; + use cron::Schedule; + use crate::api::scheduler::datetime_to_instant; + + #[actix_rt::test] + async fn test_run_scheduler() { + // Define a cron expression that runs every second + let expression = "0/1 * * * * * *"; // every second + + let runs = AtomicU8::new(0); + let run_me = || runs.fetch_add(1, Ordering::Relaxed); + + let start = std::time::Instant::now(); + match Schedule::from_str(expression) { + Ok(schedule) => { + let offset = *Local::now().offset(); + loop { + let mut upcoming = schedule.upcoming(offset).take(1); + if let Some(datetime) = upcoming.next() { + actix_rt::time::sleep_until(actix_rt::time::Instant::from(datetime_to_instant(datetime))).await; + run_me(); + } + if runs.load(Ordering::Relaxed) == 6 { + break; + } + } + } + Err(_) => {} + }; + let duration = start.elapsed(); + + assert!(runs.load(Ordering::Relaxed) == 6, "Failed to run"); + assert!(duration.as_secs() > 4, "Failed time"); + } } \ No newline at end of file diff --git a/src/api/xmltv_api.rs b/src/api/xmltv_api.rs index 4a54e9427..ad310cff9 100644 --- a/src/api/xmltv_api.rs +++ b/src/api/xmltv_api.rs @@ -17,6 +17,7 @@ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigInput, ConfigTarget}; use crate::model::config::TargetType; use crate::repository::m3u_repository::m3u_get_epg_file_path; +use crate::repository::storage::get_target_storage_path; use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path}; use crate::utils::{file_utils, request_utils}; @@ -51,10 +52,13 @@ fn get_epg_path_for_target_of_type(target_name: &str, file_path: Option } fn get_epg_path_for_target(config: &Config, target: &ConfigTarget) -> Option { + // TODO if we share the same virtual_id for epg, can we store an epg file for the target ? for output in &target.output { match output.target { TargetType::M3u => { - return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path(config, target)); + if let Some(target_path) = get_target_storage_path(config, &target.name) { + return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path(&target_path)); + } } TargetType::Xtream => { if let Some(storage_path) = xtream_get_storage_path(config, &target.name) { diff --git a/src/api/xtream_api.rs b/src/api/xtream_api.rs index 972701d69..e095affaf 100644 --- a/src/api/xtream_api.rs +++ b/src/api/xtream_api.rs @@ -1,7 +1,6 @@ // https://github.com/tellytv/go.xtream-codes/blob/master/structs.go use std::collections::HashMap; -use std::convert::TryFrom; use std::fmt::{Display, Formatter}; use std::io::{Error, ErrorKind}; use std::path::Path; @@ -17,12 +16,49 @@ use crate::api::api_utils::{get_user_server_info, get_user_target, get_user_targ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigInput, ConfigTarget}; use crate::model::config::TargetType; -use crate::model::playlist::{XtreamCluster, XtreamPlaylistItem}; +use crate::model::playlist::{PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; +use crate::repository::storage::{get_target_storage_path, hash_string}; +use crate::repository::target_id_mapping::TargetIdMapping; use crate::repository::xtream_repository; use crate::utils::{json_utils, request_utils}; +macro_rules! try_option_bad_request { + ($option:expr, $msg_is_error:expr, $msg:expr) => { + match $option { + Some(value) => value, + None => { + if $msg_is_error {error!("{}", $msg);} else {debug!("{}", $msg);} + return HttpResponse::BadRequest().finish(); + } + } + }; + ($option:expr) => { + match $option { + Some(value) => value, + None => return HttpResponse::BadRequest().finish(), + } + }; +} +macro_rules! try_result_bad_request { + ($option:expr, $msg_is_error:expr, $msg:expr) => { + match $option { + Ok(value) => value, + Err(_) => { + if $msg_is_error {error!("{}", $msg);} else {debug!("{}", $msg);} + return HttpResponse::BadRequest().finish(); + } + } + }; + ($option:expr) => { + match $option { + Ok(value) => value, + Err(_) => return HttpResponse::BadRequest().finish(), + } + }; +} + enum XtreamApiStreamContext { LiveAlt, Live, @@ -109,7 +145,6 @@ fn get_xtream_player_api_stream_url(input: &ConfigInput, context: &str, action_p } } - fn get_user_info(user: &ProxyUserCredentials, cfg: &Config) -> XtreamAuthorizationResponse { let server_info = get_user_server_info(cfg, user); @@ -157,43 +192,34 @@ async fn xtream_player_api_stream( app_state: &web::Data, stream_req: XtreamApiStreamRequest<'_>, ) -> HttpResponse { - if let Some((user, target)) = get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state) { - let target_name = &target.name; - if target.has_output(&TargetType::Xtream) { - let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id); - let req_stream_id: u32 = match FromStr::from_str(action_stream_id.trim()) { - Ok(id) => id, - Err(_) => return HttpResponse::BadRequest().finish() - }; - - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, None) { - let input_id: u16 = pli.input_id; - if let Some(input) = app_state.config.get_input_by_id(input_id) { - let mut query_path = if stream_req.action_path.is_empty() { String::new() } else { format!("{}/", stream_req.action_path) }; - query_path = format!("{query_path}{}{stream_ext}", pli.provider_id); - if let Some(stream_url) = get_xtream_player_api_stream_url(input, stream_req.context.to_string().as_str(), query_path.as_str()) { - if user.proxy == ProxyType::Redirect { - debug!("Redirecting stream request to {stream_url}"); - return HttpResponse::Found().insert_header(("Location", stream_url)).finish(); - } - return stream_response(&stream_url, req, Some(input)).await; - } - error!("Cant find stream url for target {target_name}, context {}, stream_id {req_stream_id}", stream_req.context); - } else { - error!("Cant find input for target {target_name}, context {}, stream_id {req_stream_id}", stream_req.context); - } - } else { - error!("Failed to read xtream item for stream id {}", req_stream_id); - } - } else { - debug!("Target has no xtream output {}", target_name); - } - } else { - debug!("Could not find any user {}", stream_req.username); + let (user, target) = try_option_bad_request!(get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state), false, format!("Could not find any user {}", stream_req.username)); + let target_name = &target.name; + if !target.has_output(&TargetType::Xtream) { + debug!("Target has no xtream output {}", target_name); + return HttpResponse::BadRequest().finish(); } - HttpResponse::BadRequest().finish() + let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id); + let virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); + let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None), true, format!("Failed to read xtream item for stream id {}", virtual_id)); + let input = try_option_bad_request!(app_state.config.get_input_by_id(pli.input_id), true, format!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context)); + + let query_path = if stream_req.action_path.is_empty() { + format!("{}{stream_ext}", pli.provider_id) + } else { + format!("{}/{}{stream_ext}", stream_req.action_path, pli.provider_id) + }; + + let stream_url = try_option_bad_request!(get_xtream_player_api_stream_url(input, stream_req.context.to_string().as_str(), &query_path), true, format!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context)); + + if user.proxy == ProxyType::Redirect { + debug!("Redirecting stream request to {stream_url}"); + return HttpResponse::Found().insert_header(("Location", stream_url)).finish(); + } + + stream_response(&stream_url, req, Some(input)).await } + async fn xtream_player_api_live_stream( req: HttpRequest, api_req: web::Query, @@ -263,7 +289,7 @@ async fn xtream_player_api_streaming_timeshift( fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result { if let Ok(mut doc) = serde_json::from_str::>(content) { if let Some(Value::Object(movie_data)) = doc.get_mut("movie_data") { - let stream_id = pli.stream_id; + let stream_id = pli.virtual_id; let category_id = pli.category_id; movie_data.insert("stream_id".to_string(), Value::Number(serde_json::value::Number::from(stream_id))); movie_data.insert("category_id".to_string(), Value::Number(serde_json::value::Number::from(category_id))); @@ -278,44 +304,7 @@ fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: } } } - Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.stream_id))) -} - -fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result { - if let Ok(mut doc) = serde_json::from_str::(content) { - let mut new_id_to_provider_id_mapping: Vec<(u32, u32)> = Vec::new(); - if let Some(mut new_id) = xtream_repository::xtream_get_max_series_info_episode_id(config, target.name.replace(' ', "_").as_str()) { - if let Some(episodes) = doc.get_mut("episodes") { - if let Some(episodes_map) = episodes.as_object_mut() { - let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); - for (_season, episode_list) in episodes_map { - // Iterate over items in the episode - if let Some(entries) = episode_list.as_array_mut() { - for entry in entries { - if let Some(episode) = entry.as_object_mut() { - if let Some(episode_id) = episode.get("id") { - if let Ok(provider_id) = episode_id.as_str().unwrap().parse::() { - new_id += 1; - new_id_to_provider_id_mapping.push((new_id, provider_id)); - episode.insert("id".to_string(), Value::String(new_id.to_string())); - } - } - if options.skip_series_direct_source { - episode.insert("direct_source".to_string(), Value::String(String::new())); - } - } - } - } - } - } - if let Ok(result) = serde_json::to_string(&doc) { - let _ = xtream_repository::xtream_write_series_info(config, target.name.replace(' ', "_").as_str(), pli.stream_id, &new_id_to_provider_id_mapping, &result); - return Ok(result); - } - } - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli.stream_id))) + Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.virtual_id))) } async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> Result { @@ -325,7 +314,7 @@ async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &ConfigTarget, pli: &XtreamPlaylistItem, info_url: &str, cluster: XtreamCluster) -> Result { if cluster == XtreamCluster::Series { - if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.replace(' ', "_").as_str(), pli.stream_id) { + if let Some(content) = xtream_repository::xtream_load_series_info(config, target.name.as_str(), pli.virtual_id) { return Ok(content); } } @@ -333,28 +322,24 @@ async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &C if let Ok(content) = xtream_get_stream_info_content(info_url, input).await { return match cluster { XtreamCluster::Live => Ok(content), - XtreamCluster::Video => { - return get_xtream_vod_info(target, pli, &content); - } - XtreamCluster::Series => { - return get_xtream_series_info(config, target, pli, &content); - } + XtreamCluster::Video => get_xtream_vod_info(target, pli, &content), + XtreamCluster::Series => xtream_repository::write_and_get_xtream_series_info(config, target, pli, &content), }; } Err(Error::new(std::io::ErrorKind::Other, format!("Cant find stream with id: {}/{}/{}", - target.name.replace(' ', "_").as_str(), &cluster, pli.stream_id))) + target.name.replace(' ', "_").as_str(), &cluster, pli.virtual_id))) } async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserCredentials, target: &ConfigTarget, stream_id: &str, cluster: XtreamCluster) -> HttpResponse { - let req_stream_id: u32 = match FromStr::from_str(stream_id) { + let virtual_id: u32 = match FromStr::from_str(stream_id) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().finish() }; - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, Some(cluster)) { + if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(cluster)) { let input_id = pli.input_id; if let Some(input) = app_state.config.get_input_by_id(input_id) { if let Some(info_url) = get_xtream_player_api_info_url(input, cluster, pli.provider_id) { @@ -377,12 +362,12 @@ async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserC async fn xtream_get_short_epg(app_state: &AppState, user: &ProxyUserCredentials, target: &ConfigTarget, stream_id: &str, limit: &str) -> HttpResponse { let target_name = &target.name; if target.has_output(&TargetType::Xtream) { - let req_stream_id: u32 = match FromStr::from_str(stream_id.trim()) { + let virtual_id: u32 = match FromStr::from_str(stream_id.trim()) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().finish() }; - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, None) { + if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None) { let input_id: u16 = pli.input_id; if let Some(input) = app_state.config.get_input_by_id(input_id) { if let Some(action_url) = get_xtream_player_api_action_url(input, "get_short_epg") { @@ -431,59 +416,33 @@ async fn xtream_player_api_handle_content_action(config: &Config, target_name: & } async fn xtream_get_catchup_response(app_state: &AppState, target: &ConfigTarget, stream_id: &str, start: &str, end: &str) -> HttpResponse { - let req_stream_id: u32 = match FromStr::from_str(stream_id) { - Ok(id) => id, - Err(_) => return HttpResponse::BadRequest().finish() - }; + let virtual_id: u32 = try_result_bad_request!(FromStr::from_str(stream_id)); + let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(XtreamCluster::Live))); + let input = try_option_bad_request!(app_state.config.get_input_by_id(pli.input_id)); + let info_url = try_option_bad_request!(get_xtream_player_api_action_url(input, "get_simple_data_table").map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id))); + let content = try_result_bad_request!(xtream_get_stream_info_content(info_url.as_str(), input).await); + let mut doc: Map = try_result_bad_request!(serde_json::from_str(&content)); + let epg_listings = try_option_bad_request!(doc.get_mut("epg_listings").and_then(Value::as_array_mut)); + let target_path = try_option_bad_request!(get_target_storage_path(&app_state.config, target.name.as_str())); + let mut target_id_mapping = TargetIdMapping::new(&target_path); - if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, Some(XtreamCluster::Live)) { - let input_id = pli.input_id; - if let Some(input) = app_state.config.get_input_by_id(input_id) { - if let Some(info_url) = get_xtream_player_api_action_url(input, "get_simple_data_table") - .map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id)) { - if let Ok(content) = xtream_get_stream_info_content(info_url.as_str(), input).await { - if let Ok(mut doc) = serde_json::from_str::>(content.as_str()) { - if let Some(epg_listings) = doc.get_mut("epg_listings") { - if let Some(epg_listing_list) = epg_listings.as_array_mut() { - let mapping = xtream_repository::xtream_load_catchup_id_mapping(&app_state.config, target.name.as_str()); - let mut max_id = u32::try_from(mapping.len()).unwrap(); - let mut new_id_mappings = Vec::new(); - for epg_list_value in epg_listing_list { - if let Some(epg_list_item) = epg_list_value.as_object_mut() { - // TODO epg_id - if let Some(Some(provider_id)) = epg_list_item.get("id").map(|v| v.as_str()) { - if let Ok(provider_stream_id) = &FromStr::from_str(provider_id) { - let stream_id = match mapping.get(provider_stream_id) { - None => { - max_id += 1; - new_id_mappings.push((*provider_stream_id, max_id)); - max_id - } - Some(mapped_id) => *mapped_id - }; - epg_list_item.insert("id".to_string(), Value::String(stream_id.to_string())); - } - } - } - } - if !new_id_mappings.is_empty() { - if let Err(err) = xtream_repository::xtream_write_catchup_id_mapping(&app_state.config, target.name.as_str(), &new_id_mappings) { - error!("Failed to write catchup id mapping {err}"); - return HttpResponse::BadRequest().finish(); - } - } - } - } - - if let Ok(result) = serde_json::to_string(&doc) { - return HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(result); - } - } - } - } + for epg_list_item in epg_listings.iter_mut().filter_map(Value::as_object_mut) { + // TODO epg_id + if let Some(catchup_provider_id) = epg_list_item.get("id").and_then(Value::as_str).and_then(|id| id.parse::().ok()) { + let uuid = hash_string(&format!("{}/{}", pli.url, catchup_provider_id)); + let virtual_id = target_id_mapping.insert_entry(uuid, catchup_provider_id, &PlaylistItemType::Catchup, pli.provider_id); + epg_list_item.insert("id".to_string(), Value::String(virtual_id.to_string())); } } - HttpResponse::BadRequest().finish() + if let Err(err) = target_id_mapping.persist() { + error!("Failed to write catchup id mapping {err}"); + return HttpResponse::BadRequest().finish(); + } + + match serde_json::to_string(&doc) { + Ok(result) => HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(result), + Err(_) => HttpResponse::BadRequest().finish(), + } } async fn xtream_player_api( @@ -530,9 +489,9 @@ async fn xtream_player_api( } else { let cat_id = if category_id.is_empty() { 0 } else { category_id.parse::().unwrap_or(0) }; match match action { - "get_live_streams" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Live, &app_state.config, target, cat_id), - "get_vod_streams" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Video, &app_state.config, target, cat_id), - "get_series" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Series, &app_state.config, target, cat_id), + "get_live_streams" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Live, &app_state.config, target, cat_id), + "get_vod_streams" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Video, &app_state.config, target, cat_id), + "get_series" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Series, &app_state.config, target, cat_id), _ => Err(Error::new(ErrorKind::Unsupported, format!("Cant find action: {action} for target: {target_name}"))), } { Ok(payload) => HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(payload), @@ -549,13 +508,8 @@ async fn xtream_player_api( } } _ => { - if api_req.action.is_empty() { - debug!("Paremeter action is empty!"); - HttpResponse::Unauthorized().finish() - } else { - debug!("cant find user!"); - HttpResponse::BadRequest().finish() - } + debug!("{}", if api_req.action.is_empty() { "Paremeter action is empty!" } else { "cant find user!" }); + HttpResponse::BadRequest().finish() } } } diff --git a/src/filter.rs b/src/filter.rs index f488843d4..0e6b1141e 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -194,9 +194,10 @@ impl std::fmt::Display for Filter { Filter::TypeComparison(field, item_type) => { write!(f, "{} = {}", field, match item_type { PlaylistItemType::Live => "live", - PlaylistItemType::Movie => "vod", + PlaylistItemType::Video => "vod", PlaylistItemType::Series => "series", - PlaylistItemType::SeriesInfo => "series" // yes series-info is handled as series in filter + PlaylistItemType::SeriesInfo => "series", // yes series-info is handled as series in filter + _ => "unsupported" }) } Filter::Group(stmt) => { @@ -268,8 +269,8 @@ fn get_parser_field_comparison(expr: Pair, templates: &Vec Option { if text_item_type.eq_ignore_ascii_case("live") { Some(PlaylistItemType::Live) - } else if text_item_type.eq_ignore_ascii_case("movie") || text_item_type.eq_ignore_ascii_case("vod") { - Some(PlaylistItemType::Movie) + } else if text_item_type.eq_ignore_ascii_case("movie") || text_item_type.eq_ignore_ascii_case("video") || text_item_type.eq_ignore_ascii_case("vod") { + Some(PlaylistItemType::Video) } else if text_item_type.eq_ignore_ascii_case("series") { Some(PlaylistItemType::Series) } else if text_item_type.eq_ignore_ascii_case("series-info") { diff --git a/src/main.rs b/src/main.rs index bc28432bd..ca68cf275 100644 --- a/src/main.rs +++ b/src/main.rs @@ -147,11 +147,10 @@ fn init_logger(log_level: &str) { let mut log_builder = Builder::from_default_env(); if log_level.contains('=') { - let pairs: Vec<&str> = log_level.split(',').collect(); - for pair in pairs { - let kv: Vec<&str> = pair.split('=').collect(); - if kv.len() == 2 { - log_builder.filter_module(kv[0].trim(), get_log_level(kv[1].trim())); + for pair in log_level.split(',').filter(|s| s.contains('=')) { + let mut kv_iter = pair.split('=').map(str::trim); + if let (Some(module), Some(level)) = (kv_iter.next(), kv_iter.next()) { + log_builder.filter_module(module, get_log_level(level)); } } } else { diff --git a/src/model/config.rs b/src/model/config.rs index cfdfbf5e6..1e9f84b4f 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -24,6 +24,7 @@ use crate::utils::{config_reader, file_utils}; use crate::utils::default_utils::{default_as_default, default_as_false, default_as_true, default_as_empty_list, default_as_frm, default_as_empty_map, default_as_zero_u8, default_as_two_u16}; +use crate::utils::file_lock_manager::FileLockManager; pub(crate) const MAPPER_ATTRIBUTE_FIELDS: &[&str] = &[ "name", "title", "group", "id", "chno", "logo", @@ -42,6 +43,13 @@ macro_rules! valid_property { }}; } +#[macro_export] +macro_rules! create_m3u_filter_error { + ($kind: expr, $($arg:tt)*) => { + M3uFilterError::new($kind, format!($($arg)*)) + } +} + #[macro_export] macro_rules! create_m3u_filter_error_result { ($kind: expr, $($arg:tt)*) => { @@ -802,7 +810,8 @@ pub(crate) struct Config { pub t_sources_file_path: String, #[serde(skip_serializing, skip_deserializing)] pub t_api_proxy_file_path: String, - + #[serde(skip)] + pub file_locks : Arc } impl Config { diff --git a/src/model/playlist.rs b/src/model/playlist.rs index 7ab5c4ebb..692841cf4 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -2,18 +2,17 @@ use std::cell::RefCell; use std::cmp::PartialEq; use std::fmt::{Display, Formatter}; use std::rc::Rc; -use base64::Engine; -use blake3::Hasher; -use base64::engine::general_purpose; + use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{ConfigInput, ConfigTarget}; use crate::model::xmltv::TVGuide; use crate::model::xtream::{xtream_playlistitem_to_document, XtreamMappingOptions}; -use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32, default_playlist_item_type, default_stream_cluster}; -use crate::utils::request_utils::get_base_url; +use crate::processing::m3u_parser::extract_id_from_url; +use crate::repository::storage::hash_string; +use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32}; // https://de.wikipedia.org/wiki/M3U // https://siptv.eu/howto/playlist.html @@ -44,31 +43,77 @@ pub(crate) enum XtreamCluster { Series = 3, } -impl Display for XtreamCluster { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", match self { - XtreamCluster::Live => "live", - XtreamCluster::Video => "movie", - XtreamCluster::Series => "series", - }) +impl XtreamCluster { + + pub fn as_str(&self) -> &str { + match self { + XtreamCluster::Live => "Live", + XtreamCluster::Video => "Video", + XtreamCluster::Series => "Series", + } } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +impl Default for XtreamCluster { + fn default() -> Self { + XtreamCluster::Live + } +} + +impl Display for XtreamCluster { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl TryFrom for XtreamCluster { + type Error = String; + fn try_from(item_type: PlaylistItemType) -> Result { + match item_type { + PlaylistItemType::Live => Ok(XtreamCluster::Live), + PlaylistItemType::Video => Ok(XtreamCluster::Video), + PlaylistItemType::Series => Ok(XtreamCluster::Series), + _ => Err(format!("Cant convert {item_type}")), + } + } +} + +#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[repr(u8)] pub(crate) enum PlaylistItemType { Live = 1, - Movie = 2, - Series = 3, - SeriesInfo = 4, + Video = 2, + Series = 3, // xtream series description + SeriesInfo = 4, // xtream series info fetched for series description + SeriesEpisode = 5, // from SeriesInfo parsed episodes + Catchup = 6, +} + +impl Default for PlaylistItemType { + fn default() -> Self { + PlaylistItemType::Live + } +} + +impl From for PlaylistItemType { + fn from(xtream_cluster: XtreamCluster) -> Self { + match xtream_cluster { + XtreamCluster::Live => PlaylistItemType::Live, + XtreamCluster::Video => PlaylistItemType::Video, + XtreamCluster::Series => PlaylistItemType::SeriesInfo, + } + } } impl Display for PlaylistItemType { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", match self { PlaylistItemType::Live => "live", - PlaylistItemType::Movie => "movie", + PlaylistItemType::Video => "video", PlaylistItemType::Series => "series", PlaylistItemType::SeriesInfo => "series-info", + PlaylistItemType::SeriesEpisode => "series-episode", + PlaylistItemType::Catchup => "catchup" }) } } @@ -78,11 +123,11 @@ pub(crate) trait FieldAccessor { fn set_field(&mut self, field: &str, value: &str) -> bool; } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct PlaylistItemHeader { - pub uuid: Rc, // calculated - pub stream_id: Rc, // virtual id + pub uuid: Rc<[u8; 32]>, // calculated pub id: Rc, // provider id + pub virtual_id: u32, // virtual id pub name: Rc, pub chno: Rc, pub logo: Rc, @@ -95,10 +140,9 @@ pub(crate) struct PlaylistItemHeader { pub rec: Rc, pub url: Rc, pub epg_channel_id: Option>, - #[serde(default = "default_stream_cluster")] pub xtream_cluster: XtreamCluster, pub additional_properties: Option, - #[serde(default = "default_playlist_item_type", skip_serializing, skip_deserializing)] + #[serde(skip_serializing, skip_deserializing)] pub item_type: PlaylistItemType, #[serde(default = "default_as_false", skip_serializing, skip_deserializing)] pub series_fetched: bool, // only used for series_info @@ -109,22 +153,27 @@ pub(crate) struct PlaylistItemHeader { } impl PlaylistItemHeader { - pub(crate) fn gen_uuid(&mut self) { - //let base_url = get_base_url(&self.url).unwrap_or_else(|| self.url.to_string()); - // Create a Blake3 hasher - let mut hasher = Hasher::new(); - // the url should be different for each entry. - hasher.update(self.url.as_bytes()); - // Finalize and get the hash result - let hash_bytes = hasher.finalize(); - // Encode the reduced 128-bit hash to Base62 - let mut encoded_key = String::new(); - general_purpose::STANDARD.encode_string(&hash_bytes.as_bytes(), &mut encoded_key); - self.uuid = Rc::new(encoded_key); + self.uuid = Rc::new(hash_string(&self.url)) } - pub(crate) fn get_uuid(&self) -> &str { - self.uuid.as_ref() + pub(crate) fn get_uuid(&self) -> &Rc<[u8; 32]> { + &self.uuid + } + + pub(crate) fn get_provider_id(&mut self) -> Option { + match self.id.parse::() { + Ok(id) => Some(id), + Err(_) => match extract_id_from_url(&self.url) { + Some(id) => match id.parse::() { + Ok(newid) => { + self.id = Rc::new(newid.to_string()); + Some(newid) + } + Err(_) => None, + }, + None => None, + } + } } } @@ -172,11 +221,11 @@ macro_rules! generate_field_accessor_impl_for_playlist_item_header { } } -generate_field_accessor_impl_for_playlist_item_header!(id, stream_id, name, chno, logo, logo_small, group, title, parent_code, audio_track, time_shift, rec, url;); +generate_field_accessor_impl_for_playlist_item_header!(id, /*virtual_id,*/ name, chno, logo, logo_small, group, title, parent_code, audio_track, time_shift, rec, url;); #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct M3uPlaylistItem { - pub stream_id: Rc, + pub virtual_id: u32, pub provider_id: Rc, pub name: Rc, pub chno: Rc, @@ -218,7 +267,7 @@ impl M3uPlaylistItem { #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct XtreamPlaylistItem { - pub stream_id: u32, + pub virtual_id: u32, pub provider_id: u32, pub name: Rc, pub logo: Rc, @@ -252,7 +301,7 @@ impl PlaylistItem { pub fn to_m3u(&self) -> M3uPlaylistItem { let header = self.header.borrow(); M3uPlaylistItem { - stream_id: Rc::clone(&header.stream_id), + virtual_id: header.virtual_id, provider_id: Rc::clone(&header.id), name: Rc::clone(&header.name), chno: Rc::clone(&header.chno), @@ -275,7 +324,7 @@ impl PlaylistItem { match header.id.parse::() { Ok(provider_id) => { Ok(XtreamPlaylistItem { - stream_id: header.stream_id.parse::().unwrap_or(0), + virtual_id: header.virtual_id, provider_id, name: Rc::clone(&header.name), logo: Rc::clone(&header.logo), @@ -312,14 +361,12 @@ pub(crate) struct PlaylistGroup { pub id: u32, pub title: Rc, pub channels: Vec, - #[serde(default = "default_stream_cluster", skip_serializing, skip_deserializing)] + #[serde(skip_serializing, skip_deserializing)] pub xtream_cluster: XtreamCluster, } impl PlaylistGroup { - pub(crate) fn on_load(&mut self) { self.channels.iter().for_each(|pl| pl.header.borrow_mut().gen_uuid()); } -} - +} \ No newline at end of file diff --git a/src/model/xtream.rs b/src/model/xtream.rs index ba6522615..c1c199f3b 100644 --- a/src/model/xtream.rs +++ b/src/model/xtream.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use crate::model::config::ConfigTargetOptions; use crate::model::playlist::{PlaylistItem, XtreamCluster, XtreamPlaylistItem}; -use crate::utils::default_utils::{default_as_empty_rc_str, default_as_empty_list}; +use crate::utils::default_utils::{default_as_empty_list, default_as_empty_rc_str}; const LIVE_STREAM_FIELDS: &[&str] = &[]; @@ -28,8 +28,8 @@ const SERIES_STREAM_FIELDS: &[&str] = &[ fn deserialize_number_from_string<'de, D, T: DeserializeOwned>( deserializer: D, ) -> Result, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { // we define a local enum type inside of the function // because it is untagged, serde will deserialize as the first variant @@ -73,8 +73,8 @@ fn value_to_string(v: &Value) -> Option { } fn deserialize_as_option_rc_string<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { let value: Value = Deserialize::deserialize(deserializer)?; @@ -86,8 +86,8 @@ fn deserialize_as_option_rc_string<'de, D>(deserializer: D) -> Result(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { let value: Value = Deserialize::deserialize(deserializer)?; @@ -98,8 +98,8 @@ fn deserialize_as_rc_string<'de, D>(deserializer: D) -> Result, D::Er } fn deserialize_as_string_array<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, +where + D: Deserializer<'de>, { Value::deserialize(deserializer).map(|v| match v { Value::Array(value) => Some(value_to_string_array(&value)), @@ -131,9 +131,9 @@ pub(crate) struct XtreamStream { #[serde(default, deserialize_with = "deserialize_as_rc_string")] pub category_id: Rc, #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub stream_id: Option, + pub stream_id: Option, #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub series_id: Option, + pub series_id: Option, #[serde(default = "default_as_empty_rc_str", deserialize_with = "deserialize_as_rc_string")] pub stream_icon: Rc, #[serde(default = "default_as_empty_rc_str", deserialize_with = "deserialize_as_rc_string")] @@ -217,10 +217,9 @@ macro_rules! add_i64_property_if_exists { } } - impl XtreamStream { - pub(crate) fn get_stream_id(&self) -> String { - self.stream_id.map_or_else(|| self.series_id.map_or_else(String::new, |seid| format!("{seid}")), |sid| format!("{sid}")) + pub(crate) fn get_stream_id(&self) -> u32 { + self.stream_id.unwrap_or(self.series_id.unwrap_or(0)) } pub(crate) fn get_additional_properties(&self) -> Option { @@ -315,6 +314,18 @@ pub(crate) struct XtreamSeriesInfoEpisode { pub direct_source: String, } +// impl XtreamSeriesInfoEpisode { +// pub(crate) fn get_id(&self) -> u32 { +// match self.id.parse::() { +// Ok(id) => id, +// Err(_) => { +// error!("Failed to convert id to number {}", self.id); +// 0 +// } +// } +// } +// } + #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct XtreamSeriesInfo { pub seasons: Vec, @@ -358,7 +369,7 @@ impl XtreamMappingOptions { pub fn from_target_options(options: Option<&ConfigTargetOptions>) -> Self { let (skip_live_direct_source, skip_video_direct_source, skip_series_direct_source) = options .map_or((false, false, false), |o| (o.xtream_skip_live_direct_source, - o.xtream_skip_video_direct_source, o.xtream_skip_series_direct_source)); + o.xtream_skip_video_direct_source, o.xtream_skip_series_direct_source)); Self { skip_live_direct_source, skip_video_direct_source, @@ -414,7 +425,7 @@ fn append_prepared_series_properties(add_props: Option<&Map>, doc } pub(crate) fn xtream_playlistitem_to_document(pli: &XtreamPlaylistItem, options: &XtreamMappingOptions) -> serde_json::Value { - let stream_id_value = Value::Number(serde_json::Number::from(pli.stream_id)); + let stream_id_value = Value::Number(serde_json::Number::from(pli.virtual_id)); let mut document = serde_json::Map::from_iter([ ("category_id".to_string(), Value::String(format!("{}", &pli.category_id))), ("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(pli.category_id))]))), diff --git a/src/processing/m3u_parser.rs b/src/processing/m3u_parser.rs index 6c4b05231..e366f5d7f 100644 --- a/src/processing/m3u_parser.rs +++ b/src/processing/m3u_parser.rs @@ -4,7 +4,6 @@ use std::rc::Rc; use crate::model::config::{Config, ConfigInput}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use crate::utils::default_utils::{default_as_empty_rc_str, default_playlist_item_type, default_stream_cluster}; use crate::utils::string_utils; #[inline] @@ -76,27 +75,10 @@ fn skip_digit(it: &mut std::str::Chars) -> Option { fn create_empty_playlistitem_header(input_id: u16, url: &str) -> PlaylistItemHeader { PlaylistItemHeader { - uuid: default_as_empty_rc_str(), - id: default_as_empty_rc_str(), - stream_id: default_as_empty_rc_str(), - name: default_as_empty_rc_str(), - chno: default_as_empty_rc_str(), - logo: default_as_empty_rc_str(), - logo_small: default_as_empty_rc_str(), - group: default_as_empty_rc_str(), - title: default_as_empty_rc_str(), - parent_code: default_as_empty_rc_str(), - audio_track: default_as_empty_rc_str(), - time_shift: default_as_empty_rc_str(), - rec: default_as_empty_rc_str(), url: Rc::new(url.to_owned()), - epg_channel_id: None, - item_type: default_playlist_item_type(), - xtream_cluster: default_stream_cluster(), - additional_properties: None, - series_fetched: false, category_id: 0, input_id, + ..Default::default() } } @@ -147,14 +129,14 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str plih.id = Rc::new(chanid); } } - plih.stream_id = Rc::clone(&plih.id); + // plih.virtual_id = plih.id; plih.epg_channel_id = Some(Rc::clone(&plih.id)); } if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { // TODO find Series based on group or configured names plih.xtream_cluster = XtreamCluster::Video; - plih.item_type = PlaylistItemType::Movie; + plih.item_type = PlaylistItemType::Video; } { @@ -174,7 +156,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str pub(crate) fn extract_id_from_url(url: &str) -> Option { if let Some(filename) = url.split('/').last() { - return if let Some(index) = filename.rfind('.') { + return if let Some(index) = filename.rfind('.') { Some(filename[..index].to_string()) } else { Some(filename.to_string()) @@ -183,14 +165,17 @@ pub(crate) fn extract_id_from_url(url: &str) -> Option { None } -pub(crate) fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: impl Iterator, mut visit: F) { +pub(crate) fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F) +where + I: Iterator, +{ let mut header: Option = None; let mut group: Option = None; let video_suffixes = cfg.video.as_ref().unwrap().extensions.iter().map(String::as_str).collect::>(); for line in lines { if line.starts_with("#EXTINF") { - header = Some(line); + header = Some(String::from(line)); continue; } if line.starts_with("#EXTGRP") { @@ -219,29 +204,33 @@ pub(crate) fn consume_m3u(cfg: &Config, input: &ConfigIn } } -pub(crate) fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: &[String]) -> Vec { +pub(crate) fn parse_m3u<'a, I>(cfg: &Config, input: &ConfigInput, lines: I) -> Vec +where + I: Iterator, +{ let mut sort_order: Vec> = vec![]; - let mut idx: usize = 0; + let mut sort_order_idx: usize = 0; let mut group_map: std::collections::HashMap, usize> = std::collections::HashMap::new(); - consume_m3u(cfg, input, lines.iter().cloned(), |item| { + consume_m3u(cfg, input, lines, |item| { + // keep the original sort order for groups and group the playlist items let key = Rc::clone(&item.header.borrow().group); match group_map.entry(key) { std::collections::hash_map::Entry::Vacant(v) => { - v.insert(idx); - idx += 1; + v.insert(sort_order_idx); sort_order.push(vec![item]); + sort_order_idx += 1; } std::collections::hash_map::Entry::Occupied(o) => { sort_order.get_mut(*o.get()).unwrap().push(item); } } }); - let mut grp_id = 0; let result: Vec = sort_order.drain(..).map(|channels| { + // create a group based on the first playlist item let channel = channels.first(); - let cluster = channel.map(|pli| pli.header.borrow().xtream_cluster).unwrap(); - let group_title = channel.map(|pli| Rc::clone(&pli.header.borrow().group)).unwrap(); + let (cluster, group_title) = channel.map(|pli| + (pli.header.borrow().xtream_cluster, Rc::clone(&pli.header.borrow().group))).unwrap(); grp_id += 1; PlaylistGroup { id: grp_id, xtream_cluster: cluster, title: Rc::clone(&group_title), channels } }).collect(); diff --git a/src/processing/playlist_processor.rs b/src/processing/playlist_processor.rs index 57cffdb51..7244f7ce1 100644 --- a/src/processing/playlist_processor.rs +++ b/src/processing/playlist_processor.rs @@ -25,8 +25,8 @@ use crate::processing::playlist_watch::process_group_watch; use crate::processing::xmltv_parser::flatten_tvguide; use crate::processing::xtream_processor::playlist_resolve_series; use crate::repository::playlist_repository::persist_playlist; -use crate::utils::download; use crate::utils::default_utils::default_as_default; +use crate::utils::download; fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool { let provider = ValueProvider { pli: RefCell::new(pli) }; @@ -96,7 +96,7 @@ fn playlistitem_comparator(a: &PlaylistItem, b: &PlaylistItem, channel_sort: &Co } } } - }, + } None => { let ordering = value_a.partial_cmp(&value_b).unwrap(); match channel_sort.order { @@ -255,7 +255,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) { for channel in &plg.channels { let provider = ValueProvider { pli: RefCell::new(channel) }; if counter.filter.filter(&provider, &mut mock_processor) { - let new_value = if counter.modifier == CounterModifier::Assign { + let new_value = if counter.modifier == CounterModifier::Assign { cntval.to_string() } else { let value = match channel.header.borrow_mut().get_field(&counter.field) { @@ -264,7 +264,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) { }; if counter.modifier == CounterModifier::Suffix { format!("{}{}{}", value, counter.concat, cntval.to_string()) - } else { + } else { format!("{}{}{}", cntval.to_string(), counter.concat, value) } }; @@ -295,10 +295,11 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc) -> (Vec, Vec) { let source = cfg.sources.get(source_idx).unwrap(); - let mut source_playlists = Vec::new(); - let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count(); let mut errors = vec![]; let mut stats = HashMap::::new(); + let mut source_playlists = Vec::new(); + let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count(); + // Downlod the sources for input in &source.inputs { let input_id = input.id; if is_input_enabled(enabled_inputs, input.enabled, input_id, &user_targets) { @@ -306,16 +307,16 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

download::get_m3u_playlist(&cfg, input, &cfg.working_dir).await, InputType::Xtream => download::get_xtream_playlist(input, &cfg.working_dir).await, }; + // @TODO optmization dont hold tv_guide in memory, persist raw and later use sax parser to extract. let (tvguide, mut tvguide_errors) = if error_list.is_empty() { download::get_xmltv(&cfg, input, &cfg.working_dir).await } else { (None, vec![]) }; - error_list.drain(..).for_each(|err| errors.push(err)); - tvguide_errors.drain(..).for_each(|err| errors.push(err)); + errors.extend(error_list.drain(..).chain(tvguide_errors.drain(..))); let input_name = match &input.name { + Some(name_val) => name_val.as_str(), None => input.url.as_str(), - Some(name_val) => name_val.as_str() }; let group_count = playlistgroups.len(); let channel_count = playlistgroups.iter() @@ -334,19 +335,8 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

, source_idx: usize, user_targets: Arc

{} - Err(mut err) => err.drain(..).for_each(|e| errors.push(e)) + Err(mut err) => errors.extend(err.drain(..)) } } } @@ -370,6 +360,22 @@ async fn process_source(cfg: Arc, source_idx: usize, user_targets: Arc

InputStats { + InputStats { + name: input_name.to_string(), + input_type: input_type, + error_count: error_count, + raw_stats: PlaylistStats { + group_count, + channel_count, + }, + processed_stats: PlaylistStats { + group_count: 0, + channel_count: 0, + }, + } +} + async fn process_sources(config: Arc, user_targets: Arc) -> (Vec, Vec) { let mut handle_list = vec![]; let thread_num = config.threads; @@ -390,10 +396,8 @@ async fn process_sources(config: Arc, user_targets: Arc) let (mut res_stats, mut res_errors) = System::new().block_on(async { process_source(cfg, index, usr_trgts).await }); - res_errors.drain(..) - .for_each(|err| shared_errors.lock().unwrap().push(err)); - res_stats.drain(..) - .for_each(|stat| shared_stats.lock().unwrap().push(stat)); + shared_errors.lock().unwrap().extend(res_errors.drain(..)); + shared_stats.lock().unwrap().extend(res_stats.drain(..)); }; handles.push(thread::spawn(process)); if handles.len() >= thread_num as usize { @@ -401,10 +405,8 @@ async fn process_sources(config: Arc, user_targets: Arc) } } else { let (mut res_stats, mut res_errors) = process_source(cfg, index, usr_trgts).await; - res_errors.drain(..) - .for_each(|err| shared_errors.lock().unwrap().push(err)); - res_stats.drain(..) - .for_each(|stat| shared_stats.lock().unwrap().push(stat)); + shared_errors.lock().unwrap().extend(res_errors.drain(..)); + shared_stats.lock().unwrap().extend(res_stats.drain(..)); } } for handle in handle_list { @@ -464,7 +466,8 @@ fn flatten_groups(mut playlistgroups: Vec) -> Vec } async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>], - target: &ConfigTarget, cfg: &Config, + target: &ConfigTarget, + cfg: &Config, stats: &mut HashMap, errors: &mut Vec) -> Result<(), Vec> { let pipe = get_processing_pipe(target); @@ -493,12 +496,11 @@ async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>], let mut new_epg = vec![]; new_fetched_playlists.drain(..).for_each(|mut fp| { + // collect all epg_channel ids let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) .filter_map(|c| c.header.borrow().epg_channel_id.clone()).collect(); - fp.playlistgroups.drain(..).for_each(|group| { - new_playlist.push(group); - }); + new_playlist.extend(fp.playlistgroups.drain(..)); if !epg_channel_ids.is_empty() { if let Some(tv_guide) = fp.epg { debug!("found epg information for {}", &target.name); diff --git a/src/processing/xtream_parser.rs b/src/processing/xtream_parser.rs index 80dabba4a..96246bfbe 100644 --- a/src/processing/xtream_parser.rs +++ b/src/processing/xtream_parser.rs @@ -8,12 +8,11 @@ use crate::create_m3u_filter_error_result; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::ConfigInput; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; -use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamStream}; -use crate::utils::default_utils::default_as_empty_rc_str; +use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream}; -fn map_to_xtream_category(category: &Value) -> Result, M3uFilterError> { - match serde_json::from_value::>(category.to_owned()) { - Ok(category_list) => Ok(category_list), +fn map_to_xtream_category(categories: &Value) -> Result, M3uFilterError> { + match serde_json::from_value::>(categories.to_owned()) { + Ok(xtream_categories) => Ok(xtream_categories), Err(err) => { create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "Failed to process categories {}", &err) } @@ -29,6 +28,16 @@ fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: &Value) -> Resu } } +fn create_xtream_series_info_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Rc { + if episode.direct_source.is_empty() { + let ext = episode.container_extension.clone(); + let stream_base_url = format!("{url}/series/{username}/{password}/{}.{ext}", episode.id); + Rc::new(stream_base_url) + } else { + Rc::new(episode.direct_source.clone()) + } +} + pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &ConfigInput) -> Result>, M3uFilterError> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); @@ -39,33 +48,18 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: & let result: Vec = series_info.episodes.values().flatten().map(|episode| PlaylistItem { header: RefCell::new(PlaylistItemHeader { - uuid: default_as_empty_rc_str(), - id: Rc::new(episode.id.clone()), - stream_id: Rc::new(episode.id.clone()), + id: Rc::new(episode.id.to_string()), name: Rc::new(episode.title.clone()), - chno: default_as_empty_rc_str(), logo: Rc::new(episode.info.movie_image.clone()), - logo_small: default_as_empty_rc_str(), group: Rc::new(group_title.to_string()), title: Rc::new(episode.title.clone()), - parent_code: default_as_empty_rc_str(), - audio_track: default_as_empty_rc_str(), - time_shift: default_as_empty_rc_str(), - rec: default_as_empty_rc_str(), - url: if episode.direct_source.is_empty() { - let ext = episode.container_extension.clone(); - let stream_base_url = format!("{}/series/{}/{}/{}.{}", url, username, password, episode.id.as_str(), ext); - Rc::new(stream_base_url) - } else { - Rc::new(episode.direct_source.clone()) - }, - epg_channel_id: None, + url: create_xtream_series_info_url(url, username, password, episode), item_type: PlaylistItemType::Series, xtream_cluster: XtreamCluster::Series, additional_properties: episode.get_additional_properties(&series_info), - series_fetched: false, category_id: 0, input_id: input.id, + ..Default::default() }) }).collect(); if result.is_empty() { Ok(None) } else { Ok(Some(result)) } @@ -76,69 +70,61 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: & } } +fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &str, password: &str, stream: &XtreamStream) -> Rc { + if stream.direct_source.is_empty() { + let stream_base_url = match xtream_cluster { + XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()), + XtreamCluster::Video => { + let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str()); + format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext) + } + XtreamCluster::Series => + format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}", + url, username, password, &stream.get_stream_id()) + }; + Rc::new(stream_base_url) + } else { + Rc::clone(&stream.direct_source) + } +} pub(crate) fn parse_xtream(input: &ConfigInput, xtream_cluster: XtreamCluster, - category: &Value, + categories: &Value, streams: &Value) -> Result>, M3uFilterError> { - match map_to_xtream_category(category) { - Ok(mut categories) => { + match map_to_xtream_category(categories) { + Ok(mut xtream_categories) => { let input_id = input.id; let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); return match map_to_xtream_streams(xtream_cluster, streams) { - Ok(streams) => { + Ok(xtream_streams) => { let group_map: HashMap::, RefCell> = - categories.drain(..).map(|category| + xtream_categories.drain(..).map(|category| (Rc::clone(&category.category_id), RefCell::new(category)) ).collect(); - for stream in streams { + for stream in xtream_streams { if let Some(group) = group_map.get(&stream.category_id) { let mut grp = group.borrow_mut(); let category_name = &grp.category_name; let item = PlaylistItem { header: RefCell::new(PlaylistItemHeader { - uuid: default_as_empty_rc_str(), - id: Rc::new(stream.get_stream_id()), - stream_id: Rc::new(stream.get_stream_id()), + id: Rc::new(stream.get_stream_id().to_string()), name: Rc::clone(&stream.name), - chno: default_as_empty_rc_str(), logo: Rc::clone(&stream.stream_icon), - logo_small: default_as_empty_rc_str(), group: Rc::clone(category_name), title: Rc::clone(&stream.name), - parent_code: default_as_empty_rc_str(), - audio_track: default_as_empty_rc_str(), - time_shift: default_as_empty_rc_str(), - rec: default_as_empty_rc_str(), - url: if stream.direct_source.is_empty() { - let stream_base_url = match xtream_cluster { - XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()), - XtreamCluster::Video => { - let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str()); - format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext) - } - XtreamCluster::Series => - format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}", - url, username, password, &stream.get_stream_id()) - }; - Rc::new(stream_base_url) - } else { - Rc::clone(&stream.direct_source) - }, + url: create_xtream_url(xtream_cluster, url, username, password, &stream), epg_channel_id: stream.epg_channel_id.clone(), - item_type: match xtream_cluster { - XtreamCluster::Live => PlaylistItemType::Live, - XtreamCluster::Video => PlaylistItemType::Movie, - XtreamCluster::Series => PlaylistItemType::SeriesInfo, - }, + item_type: PlaylistItemType::from(xtream_cluster), xtream_cluster, additional_properties: stream.get_additional_properties(), series_fetched: false, category_id: 0, input_id, + ..Default::default() }), }; grp.add(item); @@ -160,4 +146,5 @@ pub(crate) fn parse_xtream(input: &ConfigInput, } Err(err) => Err(err) } -} \ No newline at end of file +} + diff --git a/src/repository/bplustree.rs b/src/repository/bplustree.rs index 03127ae81..5ed74b1a0 100644 --- a/src/repository/bplustree.rs +++ b/src/repository/bplustree.rs @@ -1,25 +1,81 @@ -use std::fs::File; -use std::io::{self}; -use flate2::Compression; -use flate2::write::{GzEncoder}; -use flate2::read::{GzDecoder}; +use std::array::TryFromSliceError; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::marker::PhantomData; +use std::path::Path; +use flate2::Compression; +use log::error; use serde::{Deserialize, Serialize}; -const T: usize = 3; // Minimum degree (T), meaning each node can contain at most 2*T - 1 keys +const BINCODE_OVERHEAD: usize = 4; +const BLOCK_SIZE: usize = 4096; +const POINTER_SIZE: usize = size_of::>(); -#[derive(Serialize, Deserialize, Debug, Clone)] -struct KeyValue { - key: K, - value: V, +fn is_multiple_of_block_size(file: &File) -> io::Result { + let file_size = file.metadata()?.len(); // Get the file size in bytes + Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE } +fn is_file_valid(file: File) -> io::Result { + match is_multiple_of_block_size(&file) { + Ok(valid) => { + if !valid { + return Err(io::Error::new(io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); + } + } + Err(err) => return Err(err) + } + Ok(file) +} + +#[inline] +fn u32_from_bytes(bytes: &[u8]) -> io::Result { + Ok(u32::from_le_bytes(bytes.try_into().map_err(|e: TryFromSliceError| io::Error::new(io::ErrorKind::Other, e.to_string()))?)) +} + +#[inline] +fn bincode_serialize(value: &T) -> io::Result> +where + T: serde::Serialize, +{ + bincode::serialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) +} + +#[inline] +fn bincode_deserialize(value: &[u8]) -> io::Result +where + T: for<'a> serde::Deserialize<'a>, +{ + bincode::deserialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) +} + + +fn get_entry_index_upper_bound(keys: &Vec, key: &K) -> usize +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut left = 0; + let mut right = keys.len(); + while left < right { + let mid = left + ((right - left) >> 1); + if &keys[mid] <= key { + left = mid + 1; + } else { + right = mid; + } + } + left +} + + #[derive(Serialize, Deserialize, Debug, Clone)] struct BPlusTreeNode { keys: Vec, - children: Vec>>, + children: Vec>, is_leaf: bool, - values: Vec>, // only used in leaf nodes + values: Vec, // only used in leaf nodes } impl BPlusTreeNode @@ -27,57 +83,245 @@ where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, { + #[inline] fn new(is_leaf: bool) -> Self { BPlusTreeNode { + is_leaf, keys: vec![], children: vec![], - is_leaf, values: vec![], } } - fn insert_non_full(&mut self, key: K, value: V) { - let pos = self.keys.binary_search(&key).unwrap_or_else(|pos| pos); + #[inline] + fn is_overflow(&self, order: usize) -> bool { + self.keys.len() > order + } - if self.is_leaf { - self.keys.insert(pos, key); - self.values.insert(pos, Some(value)); + #[inline] + fn get_median_index(order: usize) -> usize { + order >> 1 + } + + fn find_leaf_entry<'a>(&self, node: &'a BPlusTreeNode) -> &'a K { + if node.is_leaf { + node.keys.get(0).unwrap() } else { - let mut idx = pos; - if self.children[pos].as_mut().unwrap().keys.len() == 2 * T - 1 { - self.split_child(pos); - if key > self.keys[pos] { - idx = pos + 1; - } - } - self.children[idx].as_mut().unwrap().insert_non_full(key, value); + let child = node.children.get(0).unwrap(); + self.find_leaf_entry(child) } } - fn split_child(&mut self, pos: usize) { - let t = T - 1; - let mut new_node = BPlusTreeNode::new(self.children[pos].as_ref().unwrap().is_leaf); - let mut old_node = self.children[pos].take().unwrap(); + #[allow(dead_code)] + fn query(&self, key: &K) -> Option<&V> { + if self.is_leaf { + return match self.keys.binary_search(&key) { + Ok(idx) => self.values.get(idx), + Err(_) => None, + }; + } + let node = self.children.get(self.get_entry_index_upper_bound(key)).unwrap(); + node.query(key) + } - self.keys.insert(pos, old_node.keys.remove(t)); - self.children.insert(pos + 1, Some(new_node.clone())); + fn get_equal_entry_index(&self, key: &K) -> Option + where + K: Ord, + { + let mut left = 0; + let mut right = self.keys.len().checked_sub(1)?; + while left <= right { + let mid = left + ((right - left) >> 1); + let mid_key = &self.keys[mid]; + if mid_key == key { + return Some(mid); + } else if mid_key > key { + right = mid.checked_sub(1)?; + } else { + left = mid + 1; + } + } + None + } - if old_node.is_leaf { - new_node.keys = old_node.keys.split_off(t); - new_node.values = old_node.values.split_off(t); + fn get_entry_index_upper_bound(&self, key: &K) -> usize { + get_entry_index_upper_bound::(&self.keys, key) + } + + fn insert(&mut self, key: K, v: V, inner_order: usize, leaf_order: usize) -> Option> { + if self.is_leaf { + if let Some(eq_entry_index) = self.get_equal_entry_index(&key) { + self.values.insert(eq_entry_index, v); + return None; + } + let pos = self.get_entry_index_upper_bound(&key); + self.keys.insert(pos, key); + self.values.insert(pos, v); + if self.is_overflow(leaf_order) { + return Some(self.split(leaf_order)); + } } else { - new_node.keys = old_node.keys.split_off(t + 1); - new_node.children = old_node.children.split_off(t + 1); + let pos = self.get_entry_index_upper_bound(&key); + let child = self.children.get_mut(pos).unwrap(); + let node = child.insert(key, v, inner_order, leaf_order); + if node.is_some() { + let leaf_key = self.find_leaf_entry(node.as_ref().unwrap()); + let idx = self.get_entry_index_upper_bound(leaf_key); + self.keys.insert(idx, leaf_key.clone()); + self.children.insert(idx + 1, node.unwrap()); + if self.is_overflow(inner_order) { + return Some(self.split(inner_order)); + } + } + } + None + } + + fn split(&mut self, order: usize) -> BPlusTreeNode { + let median = BPlusTreeNode::::get_median_index(order); + if self.is_leaf { + let mut node = BPlusTreeNode::new(true); + node.keys = self.keys.split_off(median); + node.values = self.values.split_off(median); + node + } else { + let mut node = BPlusTreeNode::new(false); + node.keys = self.keys.split_off(median + 1); + node.children = self.children.split_off(median + 1); + self.children.push(node.children.get(0).unwrap().clone()); + node + } + } + + pub(crate) fn traverse(&self, visit: &mut F) + where + F: FnMut(&Vec, &Vec), + { + if self.is_leaf { + visit(&self.keys, &self.values); + } + self.children.iter().for_each(|child| child.traverse(visit)); + } + + fn serialize_to_block(&self, file: &mut W, buffer: &mut Vec, offset: u64) -> io::Result { + let mut current_offset = offset; + let buffer_slice = &mut buffer[..]; + + // Write node type (leaf or internal) + buffer_slice[0] = if self.is_leaf { 1u8 } else { 0u8 }; + let mut write_pos = 1; + + // Serialize and write keys + let keys_encoded = bincode_serialize(&self.keys)?; + let keys_bytes_len = keys_encoded.len(); + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(keys_bytes_len as u32).to_le_bytes()); + write_pos += 4; + buffer_slice[write_pos..write_pos + keys_bytes_len].copy_from_slice(&keys_encoded); + write_pos += keys_bytes_len; + + // If leaf, serialize and write values + if self.is_leaf { + let values_encoded = bincode_serialize(&self.values)?; + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(&values_encoded)?; + let compressed_bytes = encoder.finish()?; + let values_bytes_len = compressed_bytes.len(); + buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(values_bytes_len as u32).to_le_bytes()); + write_pos += 4; + buffer_slice[write_pos..write_pos + values_bytes_len].copy_from_slice(&compressed_bytes); + write_pos += values_bytes_len; } - self.children[pos] = Some(old_node); - self.children[pos + 1] = Some(new_node); + // Write the complete buffer to file, do not optimize to real filled size, + file.seek(SeekFrom::Start(offset))?; + file.write_all(&buffer_slice[..BLOCK_SIZE])?; // use BLOCK_SIZE + current_offset += BLOCK_SIZE as u64; + + if !self.is_leaf { + let pointer_offset = offset + write_pos as u64; + let mut pointer = Vec::with_capacity(self.children.len()); + for child in &self.children { + pointer.push(current_offset); + current_offset = child.serialize_to_block(file, buffer, current_offset)?; + } + + let pointer_encoded = bincode_serialize(&pointer)?; + let pointer_bytes_len = pointer_encoded.len() as u32; + + file.seek(SeekFrom::Start(pointer_offset))?; + file.write_all(&pointer_bytes_len.to_le_bytes())?; + file.write_all(&pointer_encoded)?; + } + + Ok(current_offset) + } + + fn deserialize_from_block(file: &mut R, buffer: &mut Vec, offset: u64, nested: bool) -> io::Result<(Self, Option>)> { + file.seek(SeekFrom::Start(offset))?; + file.read_exact(buffer)?; + + // Read the node type directly from buffer + let is_leaf = buffer[0] == 1u8; + let mut read_pos = 1; + + // Deserialize keys + let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; + read_pos += 4; + let keys: Vec = bincode_deserialize(&buffer[read_pos..read_pos + keys_length])?; + read_pos += keys_length; + + // Deserialize values if leaf node + let values = if is_leaf { + let values_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; + read_pos += 4; + let compressed_bytes = &buffer[read_pos..read_pos + values_length]; + let mut decoder = flate2::write::ZlibDecoder::new(Vec::new()); + decoder.write_all(&compressed_bytes)?; + let values_bytes = decoder.finish()?; + let values: Vec = bincode_deserialize(&values_bytes)?; + read_pos += values_length; + values + } else { + vec![] + }; + + // Deserialize children indices if internal node + let (children, children_pointer) = if !is_leaf { + let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize; + read_pos += 4; + let pointers: Vec = bincode_deserialize(&buffer[read_pos..read_pos + pointers_length])?; + if nested { + let nodes: Result>, io::Error> = pointers + .iter() + .map(|pointer| { + BPlusTreeNode::::deserialize_from_block(file, buffer, *pointer, nested) + .map(|(node, _)| node) + .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string())) + }) + .collect(); + + (nodes?, None) + } else { + (vec![], Some(pointers)) + } + } else { + (vec![], None) + }; + + Ok((BPlusTreeNode { + is_leaf, + keys, + values, + children, + }, children_pointer)) } } #[derive(Serialize, Deserialize, Debug, Clone)] -struct BPlusTree { +pub(crate) struct BPlusTree { root: BPlusTreeNode, + inner_order: usize, + leaf_order: usize, } impl BPlusTree @@ -86,96 +330,287 @@ where V: Serialize + for<'de> Deserialize<'de> + Clone, { pub(crate) fn new() -> Self { + let key_size = size_of::() + POINTER_SIZE + size_of::() + BINCODE_OVERHEAD; + let inner_order = BLOCK_SIZE / key_size; + let leaf_order = BLOCK_SIZE / (key_size + size_of::() + BINCODE_OVERHEAD); BPlusTree { - root: BPlusTreeNode::new(true), + root: BPlusTreeNode::::new(true), + inner_order, + leaf_order, + } + } + + fn new_with_root(root: BPlusTreeNode::) -> Self { + let key_size = size_of::() + POINTER_SIZE + size_of::() + BINCODE_OVERHEAD; + let inner_order = BLOCK_SIZE / key_size; + let leaf_order = BLOCK_SIZE / (key_size + size_of::() + BINCODE_OVERHEAD); + BPlusTree { + root, + inner_order, + leaf_order, } } pub(crate) fn insert(&mut self, key: K, value: V) { - if self.root.keys.len() == 2 * T - 1 { - let mut new_root = BPlusTreeNode::new(false); - new_root.children.push(Some(self.root.clone())); - new_root.split_child(0); + if self.root.keys.is_empty() { + self.root.keys.push(key); + self.root.values.push(value); + return; + } + + if let Some(node) = self.root.insert(key, value, self.inner_order, self.leaf_order) { + let child_key = if node.is_leaf { + node.keys.get(0).as_ref().unwrap() + } else { + node.find_leaf_entry(&node) + }; + + let mut new_root = BPlusTreeNode::::new(false); + new_root.keys.push(child_key.clone()); + new_root.children.push(std::mem::replace(&mut self.root, BPlusTreeNode::new(true))); // `true` als Beispiel für ein Blatt + new_root.children.push(node); + self.root = new_root; } - self.root.insert_non_full(key, value); } - pub(crate) fn query(&self, key: &K) -> Option { - let mut node = &self.root; - while !node.is_leaf { - let pos = match node.keys.binary_search(key) { - Ok(pos) => return node.values[pos].clone(), - Err(pos) => pos, + #[allow(dead_code)] + pub(crate) fn query(&self, key: &K) -> Option<&V> { + self.root.query(key) + } + + pub(crate) fn serialize(&self, filepath: &Path) -> io::Result { + let mut file = OpenOptions::new().write(true).create(true).open(filepath)?; + let mut buffer = vec![0u8; BLOCK_SIZE]; + let result = self.root.serialize_to_block(&mut file, &mut buffer, 0u64); + file.flush()?; + result + } + + pub(crate) fn deserialize(filepath: &Path) -> io::Result { + let mut file = File::open(filepath)?; + if is_multiple_of_block_size(&file).is_err() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}"))); + } + let mut buffer = vec![0u8; BLOCK_SIZE]; + let (root, _) = BPlusTreeNode::deserialize_from_block(&mut file, &mut buffer, 0, true)?; + Ok(BPlusTree::new_with_root(root)) + } + + pub(crate) fn traverse(&self, mut visit: F) + where + F: FnMut(&Vec, &Vec), + { + self.root.traverse(&mut visit); + } +} + +fn query_tree(file: &mut File, key: &K) -> Option +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let mut offset = 0; + let mut buffer = vec![0u8; BLOCK_SIZE]; + loop { + match BPlusTreeNode::::deserialize_from_block(file, &mut buffer, offset, false) { + Ok((node, pointers)) => { + if node.is_leaf { + return match node.keys.binary_search(key) { + Ok(idx) => node.values.get(idx).cloned(), + Err(_) => None, + }; + } + let child_idx = get_entry_index_upper_bound::(&node.keys, key); + offset = *pointers.unwrap().get(child_idx).unwrap(); + } + Err(err) => { + error!("Failed to read id tree from file {err}"); + return None; + } + }; + } +} + +pub(crate) struct BPlusTreeQuery { + file: File, + _marker_k: PhantomData, + _marker_v: PhantomData, +} + +impl BPlusTreeQuery +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn try_new(filepath: &Path) -> io::Result { + let file = is_file_valid(File::open(filepath)?)?; + Ok(BPlusTreeQuery { + file, + _marker_k: Default::default(), + _marker_v: Default::default(), + }) + } + + pub(crate) fn query(&mut self, key: &K) -> Option { + query_tree(&mut self.file, key) + } +} + +pub(crate) struct BPlusTreeUpdate { + file: File, + _marker_k: PhantomData, + _marker_v: PhantomData, +} + +impl BPlusTreeUpdate +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn try_new(filepath: &Path) -> io::Result { + if !filepath.exists() { + return Err(io::Error::new(io::ErrorKind::NotFound, format!("File not found {}", filepath.to_str().unwrap_or("?")))); + } + let file = OpenOptions::new() + .write(true) + .read(true) + .open(filepath)?; + let file = is_file_valid(file)?; + Ok(BPlusTreeUpdate { + file, + _marker_k: Default::default(), + _marker_v: Default::default(), + }) + } + + pub(crate) fn query(&mut self, key: &K) -> Option { + query_tree(&mut self.file, key) + } + + fn serialize_node(&mut self, offset: u64, node: BPlusTreeNode) -> io::Result { + let mut buffer = vec![0u8; BLOCK_SIZE]; + let result = node.serialize_to_block(&mut self.file, &mut buffer, offset); + self.file.flush()?; + result + } + + pub(crate) fn update(&mut self, key: &K, value: V) -> io::Result { + let mut offset = 0; + let mut buffer = vec![0u8; BLOCK_SIZE]; + loop { + match BPlusTreeNode::::deserialize_from_block(&mut self.file, &mut buffer, offset, false) { + Ok((mut node, pointers)) => { + if node.is_leaf { + return match node.keys.binary_search(key) { + Ok(idx) => { + let old_value = node.values.get(idx); + if let Some(_) = old_value { + node.values[idx] = value; + return self.serialize_node(offset, node); + } + Err(io::Error::new(io::ErrorKind::NotFound, "Entry not found")) + } + Err(_) => Err(io::Error::new(io::ErrorKind::NotFound, "Entry not found")), + }; + } + let child_idx = get_entry_index_upper_bound::(&node.keys, key); + offset = *pointers.unwrap().get(child_idx).unwrap(); + } + Err(err) => { + error!("Failed to read id tree from file {err}"); + return Err(io::Error::new(io::ErrorKind::NotFound, format!("Failed to read id tree from file {err}"))); + } }; - node = node.children[pos].as_ref().unwrap(); - } - - match node.keys.binary_search(key) { - Ok(pos) => node.values[pos].clone(), - Err(_) => None, - } - } - - pub(crate) fn serialize_to_file(&self, filename: &str) -> io::Result<()> { - let file = File::create(filename)?; - let encoder = GzEncoder::new(file, Compression::default()); - match bincode::serialize_into(encoder, &self) { - Ok(()) => Ok(()), - Err(e) => { - println!("Failed to write bplustree to disk {e}"); - Ok(()) - } - } - } - - // If file exists the file is deserialized, otherwise an empty tree is returned - pub(crate) fn deserialize_from_file(filename: &str) -> Self { - match File::open(filename) { - Ok(file) => { - let decoder = GzDecoder::new(file); - let tree: BPlusTree = bincode::deserialize_from(decoder).unwrap(); - tree - } - Err(_) => BPlusTree::new() } } } + #[cfg(test)] mod tests { use std::io; - use serde::{Deserialize, Serialize}; - use crate::repository::bplustree::{BPlusTree}; + use std::path::PathBuf; + use serde::{Deserialize, Serialize}; + + use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate}; // Example usage with a simple struct - #[derive(Serialize, Deserialize, Debug, Clone)] - struct Value { + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + struct Record { id: u32, data: String, } #[test] fn insert_test() -> io::Result<()> { - let mut tree = BPlusTree::new(); - tree.insert("abc".to_string(), Value { id: 16, data: "one".to_string() }); - tree.insert("def".to_string(), Value { id: 32, data: "two".to_string() }); - tree.insert("ghi".to_string(), Value { id: 64, data: "three".to_string() }); + let mut tree = BPlusTree::::new(); + for i in 0u32..=500 { + tree.insert(i, Record { + id: i, + data: format!("Entry {i}"), + }); + } + // // Traverse the tree + // tree.traverse(|node| { + // println!("Node: {:?}", node); + // }); + + let filepath = PathBuf::from("/tmp/tree.bin"); // Serialize the tree to a file - tree.serialize_to_file("/tmp/tree.bin")?; + tree.serialize(&filepath)?; // Deserialize the tree from the file - let tree = BPlusTree::::deserialize_from_file("/tmp/tree.bin"); + tree = BPlusTree::::deserialize(&filepath)?; // Query the tree - if let Some(value) = tree.query(&("ghi".to_string())) { - println!("Found: {:?}", value); - } else { - println!("Not found"); + for i in 0u32..=500 { + let found = tree.query(&i); + assert!(found.is_some(), "Entry {} not found", i); + assert!(found.unwrap().eq(&Record { + id: i, + data: format!("Entry {i}"), + }), "Entry {} not found", i); + } + + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::try_new(&filepath)?; + for i in 0u32..=500 { + let found = tree_query.query(&i); + assert!(found.is_some(), "Entry {} not found", i); + let entry = found.unwrap(); + assert!(entry.eq(&Record { + id: i, + data: format!("Entry {i}"), + }), "Entry {} not found", i); + } + + let mut tree_update: BPlusTreeUpdate = BPlusTreeUpdate::try_new(&filepath)?; + for i in 0u32..=500 { + if let Some(record) = tree_update.query(&i) { + let new_record = Record { + id: record.id, + data: format!("Entry {}", record.id + 9000), + }; + tree_update.update(&i, new_record)?; + } else { + assert!(false, "Entry {} not found", i); + } + } + + let mut tree_query: BPlusTreeQuery = BPlusTreeQuery::try_new(&filepath)?; + for i in 0u32..=500 { + let found = tree_query.query(&i); + assert!(found.is_some(), "Entry {} not found", i); + let entry = found.unwrap(); + let expected = Record { + id: i, + data: format!("Entry {}", i + 9000), + }; + assert!(entry.eq(&expected), "Entry not equal {:?} != {:?}", entry, expected); } Ok(()) } -} \ No newline at end of file +} diff --git a/src/repository/epg_repository.rs b/src/repository/epg_repository.rs index 211930b59..528497d71 100644 --- a/src/repository/epg_repository.rs +++ b/src/repository/epg_repository.rs @@ -42,7 +42,7 @@ fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path) -> Result<(), M Ok(()) } -pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, epg: Option<&Epg>, output: &TargetOutput) -> Result<(), M3uFilterError> { +pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, target_path: &Path, epg: Option<&Epg>, output: &TargetOutput) -> Result<(), M3uFilterError> { if let Some(epg_data) = epg { match &output.target { TargetType::M3u => { @@ -51,7 +51,7 @@ pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, epg: Option<&Epg>, M3uFilterErrorKind::Notify, format!("write epg for target {} failed: No filename set", target.name))); } - if let Some(path) = m3u_get_epg_file_path(cfg, target) { + if let Some(path) = m3u_get_epg_file_path(target_path) { if log_enabled!(Level::Debug) { debug!("writing m3u epg to {}", path.to_str().unwrap_or("?")); } diff --git a/src/repository/id_mapping.rs b/src/repository/id_mapping.rs new file mode 100644 index 000000000..a64199b0a --- /dev/null +++ b/src/repository/id_mapping.rs @@ -0,0 +1,84 @@ +// This module is not included in the build + +use std::cmp::max; +use std::io::Error; +use std::path::{Path, PathBuf}; + +use log::error; +use serde::{Deserialize, Serialize}; + +use crate::repository::bplustree::BPlusTree; + +pub(crate) struct IdMapping +where + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + dirty: bool, + tree: BPlusTree, + path: PathBuf, + max_id: u32, +} + +impl IdMapping +where + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + pub(crate) fn new(path: &Path) -> Self { + let tree: BPlusTree = match BPlusTree::::deserialize(&path) { + Ok(tree) => tree, + _ => BPlusTree::::new() + }; + + let mut max_id = 0; + tree.traverse(|keys, _| { + match keys.iter().max() { + None => {} + Some(max_value) => { + max_id = max(max_id, *max_value); + } + } + }); + + IdMapping { + dirty: false, + tree, + path: path.to_path_buf(), + max_id, + } + } + + pub(crate) fn insert(&mut self, id: u32, value: V) { + self.dirty = true; + self.tree.insert(id, value); + } + + pub(crate) fn persist(&mut self) -> Result<(), Error> { + if self.dirty { + self.tree.serialize(&self.path)?; + } + self.dirty = false; + Ok(()) + } + + pub(crate) fn max_id(&self) -> u32 { + self.max_id + } + + pub(crate) fn query(&self, id: u32) -> Option<&V> { + self.tree.query(&id) + } +} + +impl Drop for IdMapping +where + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + fn drop(&mut self) { + match self.persist() { + Ok(_) => {} + Err(err) => { + error!("Failed to persist id mapping {:?} err:{}", &self.path, err.to_string()) + } + } + } +} \ No newline at end of file diff --git a/src/repository/index_record.rs b/src/repository/index_record.rs deleted file mode 100644 index 1bb0e7808..000000000 --- a/src/repository/index_record.rs +++ /dev/null @@ -1,70 +0,0 @@ -use std::fs::File; -use std::io::{Error, Read, Seek, SeekFrom, Write}; -use std::path::Path; -use crate::utils::file_utils; - -/** -We write the structs with `bincode::encode` to a file. -To access each entry we need a index file where we can find the -Entries with offset and size of the encoded struct. -This is used for the index file where first entry is the index -of the encoded file, and size is the size of the encoded struct. - -We also use it for different purposes, like storing id -> to id mapping. - */ -pub(in crate::repository) struct IndexRecord { - pub left: u32, - pub right: u32, -} - -impl IndexRecord { - pub fn from_file(file: &mut File, offset: u32) -> Result { - file.seek(SeekFrom::Start(u64::from(offset)))?; - let mut left_bytes = [0u8; 4]; - let mut right_bytes = [0u8; 4]; - file.read_exact(&mut left_bytes)?; - file.read_exact(&mut right_bytes)?; - let left = u32::from_le_bytes(left_bytes); - let right = u32::from_le_bytes(right_bytes); - Ok(IndexRecord { left, right }) - } - - pub fn to_file(path: &Path, left: u32, right: u32, append: bool) -> Result<(), Error> { - match file_utils::open_file_append(path, append) { - Ok(mut file) => { - let bytes = IndexRecord::to_bytes(left, right); - file.write_all(&bytes) - }, - Err(err) => Err(err) - } - } - - // pub fn from_bytes(bytes: &[u8], cursor: &mut usize) -> Result { - // if let Ok(index_bytes) = bytes[*cursor..*cursor + 4].try_into() { - // *cursor += 4; - // if let Ok(size_bytes) = bytes[*cursor..*cursor + 2].try_into() { - // *cursor += 2; - // let index = u32::from_le_bytes(index_bytes); - // let size = u16::from_le_bytes(size_bytes); - // return Ok(IndexRecord { index, size }); - // } - // } - // Err(Error::new(ErrorKind::Other, "Failed to read index")) - // } - - // pub fn as_bytes(&self) -> [u8; 6] { - // IndexRecord::to_bytes(self.index, self.size) - // } - - pub fn to_bytes(left: u32, right: u32) -> [u8; 8] { - let left_bytes: [u8; 4] = left.to_le_bytes(); - let right_bytes: [u8; 4] = right.to_le_bytes(); - let mut combined_bytes: [u8; 8] = [0; 8]; - combined_bytes[..4].copy_from_slice(&left_bytes); - combined_bytes[4..].copy_from_slice(&right_bytes); - combined_bytes - } - - pub fn get_record_size() -> u32 { 8 } - pub fn get_index_offset(index: u32) -> u32 { index * 8 } -} \ No newline at end of file diff --git a/src/repository/indexed_document_reader.rs b/src/repository/indexed_document_reader.rs index ee727f5d7..8c6d4a1c0 100644 --- a/src/repository/indexed_document_reader.rs +++ b/src/repository/indexed_document_reader.rs @@ -3,12 +3,30 @@ use std::fs::File; use std::io::{Error, ErrorKind, Read, Seek, SeekFrom}; use std::marker::PhantomData; use std::path::Path; +use crate::repository::bplustree::BPlusTreeQuery; +use crate::repository::indexed_document_writer::OffsetPointer; -use crate::repository::index_record::IndexRecord; +fn get_offset(index_path: &Path, doc_id: u32) -> Result { + match BPlusTreeQuery::::try_new(index_path) { + Ok(mut tree) => { + match tree.query(&doc_id) { + Some(offset) => Ok(offset as u64), + None => Err(Error::new(ErrorKind::NotFound, format!("doc_id not found {doc_id}"))), + } + }, + Err(err) => Err(err) + } +} + +fn read_content_size(main_file: &mut File) -> Result { + let mut size_bytes = [0u8; 4]; + main_file.read_exact(&mut size_bytes)?; + let buf_size = u32::from_le_bytes(size_bytes) as usize; + Ok(buf_size) +} pub(in crate::repository) struct IndexedDocumentReader { main_file: File, - index_file: File, cursor: u32, size: u32, failed: bool, @@ -17,37 +35,31 @@ pub(in crate::repository) struct IndexedDocumentReader { } impl IndexedDocumentReader { - pub fn new(main_path: &Path, index_path: &Path) -> Result, Error> { - if main_path.exists() && index_path.exists() { + pub fn new(main_path: &Path) -> Result, Error> { + if main_path.exists() { match File::open(main_path) { Ok(main_file) => { - match File::open(index_path) { - Ok(index_file) => { - let size = match index_file.metadata() { - Ok(metadata) => { - usize::try_from(metadata.len()).map_err(|err| Error::new(ErrorKind::Other, err))? - } - Err(_e) => 0, - }; - Ok(Self { - main_file, - index_file, - cursor: 0, - size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?, - failed: false, - t_buffer: Vec::new(), - t_type: PhantomData, - }) + let size = match main_file.metadata() { + Ok(metadata) => { + usize::try_from(metadata.len()).map_err(|err| Error::new(ErrorKind::Other, err))? } - Err(e) => Err(e) - } + Err(_e) => 0, + }; + + Ok(Self { + main_file, + cursor: 0, + size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?, + failed: false, + t_buffer: Vec::new(), + t_type: PhantomData, + }) } Err(e) => Err(e) } } else { - Err(Error::new(ErrorKind::NotFound, format!("File not found {} or {}", - main_path.to_str().unwrap(), - index_path.to_str().unwrap()))) + Err(Error::new(ErrorKind::NotFound, format!("File not found {}", + main_path.to_str().unwrap()))) } } @@ -59,33 +71,47 @@ impl IndexedDocumentReader { !self.failed && self.cursor < self.size } pub fn read_next(&mut self) -> Result, Error> { - if self.has_next() { - let record = IndexRecord::from_file(&mut self.index_file, self.cursor); - self.cursor += IndexRecord::get_record_size(); - match record { - Ok(index_record) => { - let offset = u64::from(index_record.left); - let buf_size = index_record.right as usize; - if self.t_buffer.len() < buf_size { - self.t_buffer.resize(buf_size, 0u8); - } - self.main_file.seek(SeekFrom::Start(offset))?; - self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?; - return match bincode::deserialize::(&self.t_buffer[0..buf_size]) { - Ok(value) => Ok(Some(value)), - Err(err) => { - self.failed = true; - Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))) - } - }; - } - Err(err) => { - self.failed = true; - return Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))); - } + if !self.has_next() { + return Ok(None); + } + // read content-size + let buf_size: usize = read_content_size(&mut self.main_file)?; + self.cursor += 4; + // resize buffer if necessary + if self.t_buffer.capacity() < buf_size { + self.t_buffer.reserve(buf_size - self.t_buffer.capacity()); + } + self.t_buffer.resize(buf_size, 0u8); + + // read content + self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?; + self.cursor += buf_size as u32; + + // deserialize buffer + return match bincode::deserialize::(&self.t_buffer[0..buf_size]) { + Ok(value) => Ok(Some(value)), + Err(err) => { + self.failed = true; + Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}"))) + } + }; + } + + pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, doc_id: u32) -> Result + { + if main_path.exists() && index_path.exists() { + // get the offset from index + let offset = crate::repository::indexed_document_reader::get_offset(index_path, doc_id)?; + let mut main_file = File::open(main_path)?; + main_file.seek(SeekFrom::Start(offset))?; + let buf_size = read_content_size(&mut main_file)?; + let mut buffer: Vec = vec![0; buf_size]; + main_file.read_exact(&mut buffer)?; + if let Ok(item) = bincode::deserialize::(&buffer) { + return Ok(item); } } - Ok(None) + Err(Error::new(ErrorKind::Other, format!("Failed to read item for id {} - {}", doc_id, main_path.to_str().unwrap()))) } } @@ -102,20 +128,3 @@ impl Iterator for IndexedDocumentReader None } } - -pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, offset: u32) -> Result - where T: ?Sized + serde::de::DeserializeOwned -{ - if main_path.exists() && index_path.exists() { - let mut index_file = File::open(index_path)?; - let mut main_file = File::open(main_path)?; - let index_record = IndexRecord::from_file(&mut index_file, offset)?; - main_file.seek(SeekFrom::Start(u64::from(index_record.left)))?; - let mut buffer: Vec = vec![0; index_record.right as usize]; - main_file.read_exact(&mut buffer)?; - if let Ok(item) = bincode::deserialize::(&buffer[..]) { - return Ok(item); - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to read item for offset {} - {}", offset, main_path.to_str().unwrap()))) -} \ No newline at end of file diff --git a/src/repository/indexed_document_writer.rs b/src/repository/indexed_document_writer.rs index 289bdca7f..a98acae7f 100644 --- a/src/repository/indexed_document_writer.rs +++ b/src/repository/indexed_document_writer.rs @@ -2,39 +2,56 @@ use std::convert::TryFrom; use std::fs::File; use std::io::{Error, ErrorKind, Write}; use std::path::PathBuf; +use log::error; +use crate::repository::bplustree::BPlusTree; -use crate::repository::index_record::IndexRecord; use crate::utils::file_utils; -use crate::utils::file_utils::create_file_tuple; +use crate::utils::file_utils::{open_file_append}; +pub(in crate::repository) type OffsetPointer = u32; + +/** +* Creates two files, +* - content +* - index +* +* Layout of content file record is: +* - content-size (u32) + content +* +* index file is a bplustree +*/ pub(in crate::repository) struct IndexedDocumentWriter { main_path: PathBuf, index_path: PathBuf, main_file: File, - index_file: File, - main_offset: u32, - index_offset: u32, + main_offset: OffsetPointer, + index_tree: BPlusTree, } impl IndexedDocumentWriter { fn new_with_mode(main_path: PathBuf, index_path: PathBuf, append: bool) -> Result { - match create_file_tuple(&main_path, &index_path, append) { - Ok((main_file, index_file)) => { + match open_file_append(&main_path, append) { + Ok(main_file) => { let main_offset = match &main_file.metadata() { Ok(meta) => u32::try_from(meta.len()).map_err(|err| Error::new(ErrorKind::Other, err))?, Err(_) => 0 }; - let index_offset = match &index_file.metadata() { - Ok(meta) => u32::try_from(meta.len()).map_err(|err| Error::new(ErrorKind::Other, err))?, - Err(_) => 0 + + let index_tree = if append && index_path.exists() { + BPlusTree::::deserialize(&index_path).unwrap_or_else(|err| { + error!("Failed to load index {:?} {err}", index_path); + BPlusTree::::new() + }) + } else { + BPlusTree::::new() }; + Ok(Self { main_path, index_path, main_file, - index_file, main_offset, - index_offset, + index_tree }) } Err(e) => Err(e) @@ -49,27 +66,29 @@ impl IndexedDocumentWriter { Self::new_with_mode(main_path, index_path, true) } - pub fn write_doc(&mut self, doc: &T) -> Result<(u32, u32), Error> + pub fn flush(&mut self) -> std::io::Result<()> { + self.main_file.flush()?; + self.index_tree.serialize(&self.index_path).map(|_| ()) + } + + pub fn write_doc(&mut self, doc_id: u32, doc: &T) -> Result<(), Error> where T: ?Sized + serde::Serialize { - let current_main_index = self.main_offset; - let current_index_index = self.index_offset; if let Ok(encoded) = bincode::serialize(doc) { - match file_utils::check_write(&self.main_file.write_all(&encoded)) { + let content_bytes_len = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; + let mut data: Vec = content_bytes_len.to_le_bytes().to_vec(); + data.extend(&encoded); + match file_utils::check_write(&self.main_file.write_all(&data)) { Ok(()) => { - let bytes_written = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; - let combined_bytes = IndexRecord::to_bytes(self.main_offset, bytes_written); - if let Err(err) = file_utils::check_write(&self.index_file.write_all(&combined_bytes)) { - return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.index_path.to_str().unwrap(), err))); - } - self.main_offset += bytes_written; - self.index_offset += IndexRecord::get_record_size(); + self.index_tree.insert(doc_id, self.main_offset); + let written_bytes = u32::try_from(data.len()).map_err(|err| Error::new(ErrorKind::Other, err))?; + self.main_offset += written_bytes; } Err(err) => { return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.main_path.to_str().unwrap(), err))); } } } - Ok((current_main_index, current_index_index)) + Ok(()) } } \ No newline at end of file diff --git a/src/repository/m3u_repository.rs b/src/repository/m3u_repository.rs index 4f96daa97..b6028de9d 100644 --- a/src/repository/m3u_repository.rs +++ b/src/repository/m3u_repository.rs @@ -1,33 +1,32 @@ use std::fs::File; use std::io::{BufWriter, Error, ErrorKind, Write}; use std::path::{Path, PathBuf}; -use std::rc::Rc; use log::error; -use crate::{create_m3u_filter_error_result}; use crate::api::api_utils::get_user_server_info; +use crate::create_m3u_filter_error; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::api_proxy::{ProxyType, ProxyUserCredentials}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{M3uPlaylistItem, PlaylistGroup, PlaylistItem, PlaylistItemType}; -use crate::repository::index_record::IndexRecord; -use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; +use crate::repository::indexed_document_reader::IndexedDocumentReader; use crate::repository::indexed_document_writer::IndexedDocumentWriter; +use crate::repository::storage::ensure_target_storage_path; use crate::utils::file_utils; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err) + create_m3u_filter_error!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err) } } -fn m3u_get_base_file_path(cfg: &Config, target: &ConfigTarget) -> Option { - file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(format!("m3u_{}.db", target.name.replace(' ', "_").as_str())))) +fn m3u_get_base_file_path(target_path: &Path) -> Option { + Some(target_path.join(PathBuf::from("m3u.db"))) } -pub(crate) fn m3u_get_file_paths(cfg: &Config, target: &ConfigTarget) -> Option<(PathBuf, PathBuf)> { - match m3u_get_base_file_path(cfg, target) { +pub(crate) fn m3u_get_file_paths(target_path: &Path) -> Option<(PathBuf, PathBuf)> { + match m3u_get_base_file_path(target_path) { Some(m3u_path) => { let extension = m3u_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or(""))); let index_path = m3u_path.with_extension(format!("{}idx", &extension.unwrap_or(String::new()))); @@ -37,93 +36,114 @@ pub(crate) fn m3u_get_file_paths(cfg: &Config, target: &ConfigTarget) -> Option< } } -pub(crate) fn m3u_get_epg_file_path(cfg: &Config, target: &ConfigTarget) -> Option { - m3u_get_base_file_path(cfg, target) +pub(crate) fn m3u_get_epg_file_path(target_path: &Path) -> Option { + m3u_get_base_file_path(target_path) .map(|path| file_utils::add_prefix_to_filename(&path, "epg_", Some("xml"))) } -pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { - if !new_playlist.is_empty() { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(cfg, target) { +fn persist_m3u_playlist_as_text(target: &ConfigTarget, cfg: &Config, m3u_playlist: &Vec) { + if let Some(filename) = target.get_m3u_filename() { + if let Some(m3u_filename) = file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) { + match File::create(&m3u_filename) { + Ok(file) => { + let mut buf_writer = BufWriter::new(file); + let _ = buf_writer.write("#EXTM3U\n".as_bytes()); + for m3u in m3u_playlist { + let _ = buf_writer.write(m3u.to_m3u(target, None).as_bytes()); + let _ = buf_writer.write("\n".as_bytes()); + } + } + Err(_) => { + error!("Can't write m3u plain playlist {}", &m3u_filename.to_str().unwrap()); + } + } + } + } +} +pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, target_path: &Path, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> { + if !new_playlist.is_empty() { + if let Some((m3u_path, idx_path)) = m3u_get_file_paths(target_path) { let m3u_playlist = new_playlist.iter() .flat_map(|pg| &pg.channels) .filter(|&pli| pli.header.borrow().item_type != PlaylistItemType::SeriesInfo) .map(PlaylistItem::to_m3u).collect::>(); - if let Some(filename) = target.get_m3u_filename() { - if let Some(m3u_filename) = file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) { - match File::create(&m3u_filename) { - Ok(file) => { - let mut buf_writer = BufWriter::new(file); - let _ = buf_writer.write("#EXTM3U\n".as_bytes()); - for m3u in &m3u_playlist { - let _ = buf_writer.write(m3u.to_m3u(target, None).as_bytes()); - let _ = buf_writer.write("\n".as_bytes()); + persist_m3u_playlist_as_text(target, cfg, &m3u_playlist); + { + let _file_lock = cfg.file_locks.write_lock(&m3u_path).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, format!("{}", err)))?; + match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) { + Ok(mut writer) => { + for m3u in m3u_playlist { + match writer.write_doc(m3u.virtual_id, &m3u) { + Ok(_) => {} + Err(err) => return Err(cant_write_result!(&m3u_path, err)) } } - Err(_) => { - error!("Can't write m3u plain playlist {}", &m3u_filename.to_str().unwrap()); - } + writer.flush().map_err(|err| cant_write_result!(&m3u_path, err))?; } + Err(err) => return Err(cant_write_result!(&m3u_path, err)) } } - match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) { - Ok(mut writer) => { - let mut stream_id: u32 = 1; - for mut m3u in m3u_playlist { - m3u.stream_id = Rc::new(stream_id.to_string()); - match writer.write_doc(&m3u) { - Ok(_) => stream_id += 1, - Err(err) => return cant_write_result!(&m3u_path, err) - } - } - } - Err(err) => return cant_write_result!(&m3u_path, err) - } } } Ok(()) } pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, user: &ProxyUserCredentials) -> Option { - if let Some((m3u_path, idx_path)) = m3u_get_file_paths(cfg, target) { - match IndexedDocumentReader::::new(&m3u_path, &idx_path) { - Ok(mut reader) => { - let server_info = get_user_server_info(cfg, user); - let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); - let mut result = vec![]; - result.push("#EXTM3U".to_string()); - for m3u_pli in reader.by_ref() { - match user.proxy { - ProxyType::Reverse => { - let stream_id = Rc::clone(&m3u_pli.stream_id); - result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{stream_id}").as_str()))); + match ensure_target_storage_path(cfg, target.name.as_str()) { + Ok(target_path) => { + if let Some((m3u_path, _)) = m3u_get_file_paths(&target_path) { + { + let _file_lock = cfg.file_locks.read_lock(&m3u_path).map_err(|err| { + error!("Could not lock document {:?}: {}", m3u_path, err); + Error::new(ErrorKind::Other, format!("Document Reader error for target {}", &target.name)) + }).ok()?; + + match IndexedDocumentReader::::new(&m3u_path) { + Ok(mut reader) => { + let server_info = get_user_server_info(cfg, user); + let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password); + let mut result = vec![]; + result.push("#EXTM3U".to_string()); + for m3u_pli in reader.by_ref() { + match user.proxy { + ProxyType::Reverse => { + result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{}", m3u_pli.virtual_id).as_str()))); + } + ProxyType::Redirect => { + result.push(m3u_pli.to_m3u(target, None)); + } + } + }; + if reader.by_ref().has_error() { + error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap()); + } else { + return Some(result.join("\n")); + } } - ProxyType::Redirect => { - result.push(m3u_pli.to_m3u(target, None)); + Err(err) => { + error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err); } } - }; - if reader.by_ref().has_error() { - error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap()); - } else { - return Some(result.join("\n")); } - } - Err(err) => { - error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err); + } else { + error!("Could not open files for target {}", &target.name); } } - } else { - error!("Could not open files for target {}", &target.name); + Err(err) => { + error!("Could not find storage path for target {} - {}", target.name.as_str(), err); + } } None } -pub(crate) fn m3u_get_item_for_stream_id(stream_id: u32, m3u_path: &Path, idx_path: &Path) -> Result { +pub(crate) fn m3u_get_item_for_stream_id(cfg: &Config, stream_id: u32, m3u_path: &Path, idx_path: &Path) -> Result { if stream_id < 1 { return Err(Error::new(ErrorKind::Other, "id should start with 1")); } - read_indexed_item::(m3u_path, idx_path, IndexRecord::get_index_offset(stream_id - 1)) + { + let _file_lock = cfg.file_locks.read_lock(&m3u_path)?; + IndexedDocumentReader::::read_indexed_item(m3u_path, idx_path, stream_id) + } } \ No newline at end of file diff --git a/src/repository/mod.rs b/src/repository/mod.rs index a1f08200b..7db3f6877 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -2,10 +2,10 @@ pub(crate) mod playlist_repository; pub(crate) mod m3u_repository; pub(crate) mod xtream_repository; pub(crate) mod epg_repository; -pub(crate)mod kodi_repository; +pub(crate) mod kodi_repository; +pub(crate) mod storage; -mod bplustree; - -mod index_record; mod indexed_document_writer; -mod indexed_document_reader; \ No newline at end of file +mod indexed_document_reader; +pub(crate) mod target_id_mapping; +pub(crate) mod bplustree; diff --git a/src/repository/playlist_repository.rs b/src/repository/playlist_repository.rs index 3a820f793..298369eed 100644 --- a/src/repository/playlist_repository.rs +++ b/src/repository/playlist_repository.rs @@ -1,32 +1,76 @@ -use crate::m3u_filter_error::M3uFilterError; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget, TargetType}; use crate::model::playlist::PlaylistGroup; use crate::model::xmltv::Epg; use crate::repository::epg_repository::epg_write; use crate::repository::kodi_repository::kodi_write_strm_playlist; use crate::repository::m3u_repository::m3u_write_playlist; +use crate::repository::storage::{ensure_target_storage_path, get_target_id_mapping_file}; +use crate::repository::target_id_mapping::TargetIdMapping; use crate::repository::xtream_repository::xtream_write_playlist; pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg>, target: &ConfigTarget, cfg: &Config) -> Result<(), Vec> { let mut errors = vec![]; - for output in &target.output { - match match output.target { - TargetType::M3u => m3u_write_playlist(target, cfg, playlist), - TargetType::Xtream => xtream_write_playlist(target, cfg, playlist), - TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), - } { - Ok(()) => { - if !playlist.is_empty() { - match epg_write(target, cfg, epg, output) { - Ok(()) => {} - Err(err) => errors.push(err) + + match ensure_target_storage_path(cfg, target.name.as_str()) { + Ok(target_path) => { + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + match cfg.file_locks.write_lock(&target_id_mapping_file) { + Ok(_file_lock) => { + // reassign old virtual id or assign a new one + let mut target_id_mapping = TargetIdMapping::new(&target_id_mapping_file); + for group in &mut *playlist { + for channel in &group.channels { + let mut header = channel.header.borrow_mut(); + match header.get_provider_id() { + Some(provider_id) => { + let uuid = header.get_uuid(); + let item_type = header.item_type; + header.virtual_id = target_id_mapping.insert_entry(**uuid, provider_id, &item_type, 0); + } + None => { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Playlistitem has no provider id: {}", &header.title))); + } + } + } + } + + for output in &target.output { + match match output.target { + TargetType::M3u => m3u_write_playlist(target, cfg, &target_path, playlist), + TargetType::Xtream => xtream_write_playlist(target, cfg, playlist), + TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename), + } { + Ok(()) => { + if let Err(err) = target_id_mapping.persist() { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } + if !playlist.is_empty() { + match epg_write(target, cfg, &target_path, epg, output) { + Ok(()) => {} + Err(err) => errors.push(err) + } + } + } + Err(err) => errors.push(err) + } + } + if let Err(err) = target_id_mapping.persist() { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); + } + } + Err(err) => { + errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string())); } } } - Err(err) => errors.push(err) } - } + Err(err) => { + errors.push(err); + } + }; if errors.is_empty() { Ok(()) } else { Err(errors) } } diff --git a/src/repository/storage.rs b/src/repository/storage.rs new file mode 100644 index 000000000..a2ed476b9 --- /dev/null +++ b/src/repository/storage.rs @@ -0,0 +1,30 @@ +use std::path::{Path, PathBuf}; +use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; +use crate::model::config::Config; +use crate::utils::file_utils; + +pub(crate) fn hash_string(url: &str) -> [u8; 32] { + let hash = blake3::hash(url.as_bytes()); + hash.into() // Konvertiere den Hash in ein Array mit fester Größe +} + +pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf { + target_path.join(PathBuf::from("id_mapping.db")) +} + +pub(crate) fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result { + if let Some(path) = get_target_storage_path(cfg, target_name) { + if std::fs::create_dir_all(&path).is_err() { + let msg = format!("Failed to save target data, can't create directory {}", &path.to_str().unwrap()); + return Err(M3uFilterError::new(M3uFilterErrorKind::Notify, msg)); + } + Ok(path) + } else { + let msg = format!("Failed to save target data, can't create directory for target {target_name}"); + Err(M3uFilterError::new(M3uFilterErrorKind::Notify, msg)) + } +} + +pub(crate) fn get_target_storage_path(cfg: &Config, target_name: &str) -> Option { + file_utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(' ', "_")))) +} diff --git a/src/repository/target_id_mapping.rs b/src/repository/target_id_mapping.rs new file mode 100644 index 000000000..5e204c124 --- /dev/null +++ b/src/repository/target_id_mapping.rs @@ -0,0 +1,107 @@ +use std::cmp::max; +use std::collections::BTreeMap; +use std::io::Error; +use std::path::{Path, PathBuf}; + +use chrono::Local; +use log::error; +use serde::{Deserialize, Serialize}; + +use crate::model::playlist::PlaylistItemType; +use crate::repository::bplustree::BPlusTree; + +const EXPIRATION_DURATION: i64 = 86400; + +#[derive(Serialize, Deserialize, Clone)] +pub(crate) struct VirtualIdRecord { + pub virtual_id: u32, + pub provider_id: u32, + pub uuid: [u8; 32], + pub item_type: PlaylistItemType, + pub parent_virtual_id: u32, // only for series to hold series info id. + pub last_updated: i64, +} + +impl VirtualIdRecord { + fn new(provider_id: u32, virtual_id: u32, item_type: PlaylistItemType, parent_virtual_id: u32, uuid: [u8; 32]) -> Self { + let last_updated = Local::now().timestamp(); + VirtualIdRecord { provider_id, virtual_id, uuid, item_type, parent_virtual_id, last_updated } + } + + pub(crate) fn is_expired(&self) -> bool { + (Local::now().timestamp() - self.last_updated) > EXPIRATION_DURATION + } + + pub(crate) fn copy_update_timestamp(&self) -> Self { + Self::new(self.provider_id, self.virtual_id, self.item_type, self.parent_virtual_id, self.uuid) + } +} + +pub(crate) struct TargetIdMapping { + dirty: bool, + virtual_id_counter: u32, + by_virtual_id: BPlusTree, + by_uuid: BTreeMap<[u8; 32], u32>, + path: PathBuf, +} + +impl TargetIdMapping { + pub(crate) fn new(path: &Path) -> Self { + let tree_virtual_id: BPlusTree = match BPlusTree::::deserialize(&path) { + Ok(tree) => tree, + _ => BPlusTree::::new() + }; + let mut tree_uuid = BTreeMap::new(); + let mut virtual_id_counter: u32 = 0; + tree_virtual_id.traverse(|keys, values| { + match keys.iter().max() { + None => {} + Some(max_value) => { + virtual_id_counter = max(virtual_id_counter, *max_value); + } + } + values.iter().for_each(|v| { + tree_uuid.insert(v.uuid, v.virtual_id); + }); + }); + TargetIdMapping { + dirty: false, + virtual_id_counter, + by_virtual_id: tree_virtual_id, + by_uuid: tree_uuid, + path: path.to_path_buf(), + } + } + + pub(crate) fn insert_entry(&mut self, uuid: [u8; 32], provider_id: u32, item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 { + match self.by_uuid.get(&uuid) { + None => { + self.dirty = true; + self.virtual_id_counter += 1; + let record = VirtualIdRecord::new(provider_id, self.virtual_id_counter, *item_type, parent_virtual_id, uuid); + self.by_virtual_id.insert(self.virtual_id_counter, record); + self.virtual_id_counter + } + Some(record) => *record + } + } + + pub(crate) fn persist(&mut self) -> Result<(), Error> { + if self.dirty { + self.by_virtual_id.serialize(&self.path)?; + } + self.dirty = false; + Ok(()) + } +} + +impl Drop for TargetIdMapping { + fn drop(&mut self) { + match self.persist() { + Ok(_) => {} + Err(err) => { + error!("Failed to persist target id mapping {:?} err:{}", &self.path, err.to_string()) + } + } + } +} \ No newline at end of file diff --git a/src/repository/xtream_repository.rs b/src/repository/xtream_repository.rs index a3c362712..bd0327aed 100644 --- a/src/repository/xtream_repository.rs +++ b/src/repository/xtream_repository.rs @@ -1,35 +1,30 @@ use std::collections::HashMap; -use std::convert::TryInto; use std::fs::File; -use std::io::{BufReader, Error, ErrorKind, Read, Seek, SeekFrom, Write}; +use std::io::{BufReader, Error, ErrorKind}; use std::path::{Path, PathBuf}; -use std::rc::Rc; use log::error; use serde_json::{json, Value}; -use crate::create_m3u_filter_error_result; +use crate::{create_m3u_filter_error, create_m3u_filter_error_result}; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget}; use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; use crate::model::xtream::XtreamMappingOptions; -use crate::processing::m3u_parser::extract_id_from_url; -use crate::repository::index_record::IndexRecord; -use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item}; +use crate::repository::bplustree::{BPlusTreeQuery, BPlusTreeUpdate}; +use crate::repository::indexed_document_reader::IndexedDocumentReader; use crate::repository::indexed_document_writer::IndexedDocumentWriter; -use crate::utils::file_utils; +use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path, hash_string}; +use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord}; use crate::utils::json_utils::{json_iter_array, json_write_documents_to_file}; pub(crate) static COL_CAT_LIVE: &str = "cat_live"; pub(crate) static COL_CAT_SERIES: &str = "cat_series"; pub(crate) static COL_CAT_VOD: &str = "cat_vod"; -pub(crate) static COL_LIVE: &str = "live"; -pub(crate) static COL_SERIES: &str = "series"; -pub(crate) static COL_VOD: &str = "vod"; macro_rules! cant_write_result { ($path:expr, $err:expr) => { - create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write xtream playlist: {} - {}", $path.to_str().unwrap() ,$err) + create_m3u_filter_error!(M3uFilterErrorKind::Notify, "failed to write xtream playlist: {} - {}", $path.to_str().unwrap() ,$err) } } @@ -37,27 +32,6 @@ fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } -pub(crate) fn xtream_get_stream_id_cluster_index_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("cluster_index.db") -} - -// Maps episode id -> to provider_episode_id and series_id -fn xtream_get_series_episode_id_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("mapping_episode.db") -} - -// maps series_id to the index of series_info index -// direct access is not possible because the series_id is not ascending, it is random -fn xtream_get_series_id_series_info_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("mapping_series_info.db_idx") -} - -// maps catchup_id to provider_id -// direct access is not possible because the series_id is not ascending, it is random -fn xtream_get_catchup_id_mapping_file_path(storage_path: &Path) -> PathBuf { - storage_path.join("mapping_catchup.db") -} - fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { if let Some(path) = xtream_get_storage_path(cfg, target_name) { if std::fs::create_dir_all(&path).is_err() { @@ -73,7 +47,7 @@ fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result Option<(PathBuf, PathBuf)> { if cluster == XtreamCluster::Series { - let xtream_path = storage_path.join("series_info.db"); + let xtream_path = storage_path.join("series_episodes.db"); let extension = xtream_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or(""))); let index_path = xtream_path.with_extension(format!("{}idx", &extension.unwrap_or_default())); return Some((xtream_path, index_path)); @@ -81,80 +55,33 @@ fn xtream_get_info_file_paths(storage_path: &Path, cluster: XtreamCluster) -> Op None } -fn xtream_clear_series_info(storage_path: &Path) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(storage_path, XtreamCluster::Series) { - let _ = std::fs::remove_file(info_path); - let _ = std::fs::remove_file(idx_path); - let _ = std::fs::remove_file(xtream_get_series_episode_id_mapping_file_path(storage_path)); - let _ = std::fs::remove_file(xtream_get_series_id_series_info_mapping_file_path(storage_path)); - } -} - -fn xtream_clear_catchup(storage_path: &Path) { - let _ = std::fs::remove_file(xtream_get_catchup_id_mapping_file_path(storage_path)); -} - -fn write_playlist_to_file(storage_path: &Path, stream_id: &mut u32, cluster: XtreamCluster, playlist: &mut [PlaylistItem]) -> Result<(), M3uFilterError> { - let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); - match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { - Ok(mut writer) => { - for pli in playlist.iter_mut() { - if let Ok(mut xtream) = pli.to_xtream() { - xtream.stream_id = *stream_id; - match writer.write_doc(&xtream) { - Ok(_) => *stream_id += 1, - Err(err) => return cant_write_result!(&xtream_path, err) - } - } - } - if cluster == XtreamCluster::Live { - xtream_clear_catchup(storage_path); - } else if cluster == XtreamCluster::Series { - xtream_clear_series_info(storage_path); - } - Ok(()) - } - Err(err) => cant_write_result!(&xtream_path, err) - } -} - -fn save_stream_id_cluster_mapping(storage_path: &Path, id_data: &mut Vec<(XtreamCluster, u32, u32)>) -> Result<(), Error> { - let stream_id_path = xtream_get_stream_id_cluster_index_file_path(storage_path); - id_data.sort_by(|(_, start_a, _), (_, start_b, _)| start_b.cmp(start_a)); - let encoded: Vec = bincode::serialize(id_data).unwrap(); - std::fs::write(stream_id_path, encoded) -} - -fn load_stream_id_cluster_mapping(storage_path: &Path) -> Option> { - let path = xtream_get_stream_id_cluster_index_file_path(storage_path); - if path.exists() { - match std::fs::read(path) { - Ok(encoded) => { - let decoded: Vec<(XtreamCluster, u32, u32)> = bincode::deserialize(&encoded[..]).unwrap(); - Some(decoded) - } - Err(_) => None, - } - } else { - None - } -} - -fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { - let mut id_list: Vec<(XtreamCluster, u32, u32)> = vec![]; - let mut stream_id: u32 = 1; +fn write_playlists_to_file(cfg: &Config, storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> { for (cluster, playlist) in collections { - let start = stream_id; - write_playlist_to_file(storage_path, &mut stream_id, cluster, playlist)?; - id_list.push((cluster, start, stream_id)); - } - match save_stream_id_cluster_mapping(storage_path, &mut id_list) { - Ok(()) => Ok(()), - Err(err) => Err(M3uFilterError::new(M3uFilterErrorKind::Notify, format!("failed to write xtream playlist: {} - {}", storage_path.to_str().unwrap(), err))) + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, &cluster); + { + let _file_lock = cfg.file_locks.write_lock(&xtream_path).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, format!("{}", err)))?; + match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) { + Ok(mut writer) => { + for item in playlist { + match item.to_xtream() { + Ok(xtream) => { + match writer.write_doc(item.header.borrow().virtual_id, &xtream) { + Ok(_) => {} + Err(err) => return Err(cant_write_result!(&xtream_path, err)) + } + } + Err(err) => return Err(cant_write_result!(&xtream_path, err)) + } + } + writer.flush().map_err(|err| cant_write_result!(&xtream_path, err))?; + } + Err(err) => return Err(cant_write_result!(&xtream_path, err)) + } + } } + Ok(()) } - fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Option { if let Some(value) = map.get(key) { if let Some(result) = value.as_str() { @@ -193,26 +120,33 @@ fn load_old_category_ids(path: &Path) -> (u32, HashMap) { } pub(crate) fn xtream_get_storage_path(cfg: &Config, target_name: &str) -> Option { - file_utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(' ', "_")))) + match get_target_storage_path(cfg, target_name) { + Some(target_path) => Some(target_path.join(std::path::PathBuf::from("xtream"))), + None => None, + } } pub(crate) fn xtream_get_epg_file_path(path: &Path) -> PathBuf { path.join("epg.xml") } -pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: XtreamCluster) -> (PathBuf, PathBuf) { - let xtream_path = storage_path.join(format!("{}.db", match cluster { - XtreamCluster::Live => COL_LIVE, - XtreamCluster::Video => COL_VOD, - XtreamCluster::Series => COL_SERIES - })); +fn xtream_get_file_paths_for_name(storage_path: &Path, name: &str) -> (PathBuf, PathBuf) { + let xtream_path = storage_path.join(format!("{}.db", name)); let extension = xtream_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or(""))); let index_path = xtream_path.with_extension(format!("{}idx", &extension.unwrap_or_default())); (xtream_path, index_path) } +pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: &XtreamCluster) -> (PathBuf, PathBuf) { + xtream_get_file_paths_for_name(storage_path, &cluster.as_str().to_lowercase()) +} + +pub(crate) fn xtream_get_file_paths_for_series(storage_path: &Path) -> (PathBuf, PathBuf) { + xtream_get_file_paths_for_name(storage_path, "series") +} + pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut [PlaylistGroup]) -> Result<(), M3uFilterError> { - match ensure_xtream_storage_path(cfg, target.name.replace(' ', "_").as_str()) { + match ensure_xtream_storage_path(cfg, target.name.as_str()) { Ok(path) => { let mut cat_live_col = vec![]; let mut cat_series_col = vec![]; @@ -249,30 +183,8 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis let col = if header.item_type == PlaylistItemType::Series { None } else { - if header.id.parse::().is_err() { - let id_from_url = match extract_id_from_url(&header.url) { - Some(id) => match id.parse::() { - Ok(newid) => Some(newid), - Err(_) => None, - }, - None => None, - }; - - let has_id = match id_from_url { - Some(newid) => { - header.id = Rc::new(newid.to_string()); - Ok(()) - } - None => { - let title = header.title.as_str(); - errors.push(format!("Channel does not have an id: {title}")); - Err(()) - } - }; - // Instead of returning from the function, handle the error by assigning None to col. - if has_id.is_err() { - None - } else { + match header.get_provider_id() { + Some(_) => { header.category_id = *cat_id; Some(match header.xtream_cluster { XtreamCluster::Live => &mut live_col, @@ -280,13 +192,11 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis XtreamCluster::Video => &mut vod_col, }) } - } else { - header.category_id = *cat_id; - Some(match header.xtream_cluster { - XtreamCluster::Live => &mut live_col, - XtreamCluster::Series => &mut series_col, - XtreamCluster::Video => &mut vod_col, - }) + None => { + let title = header.title.as_str(); + errors.push(format!("Channel does not have an id: {title}")); + None + } } }; drop(header); @@ -309,7 +219,7 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis } } - match write_playlists_to_file(&path, vec![ + match write_playlists_to_file(cfg, &path, vec![ (XtreamCluster::Live, &mut live_col), (XtreamCluster::Video, &mut vod_col), (XtreamCluster::Series, &mut series_col)]) { @@ -338,235 +248,222 @@ pub(crate) fn xtream_get_collection_path(cfg: &Config, target_name: &str, collec Err(Error::new(ErrorKind::Other, format!("Cant find collection: {target_name}/{collection_name}"))) } -fn _xtream_get_item_for_stream_id(stream_id: u32, storage_path: &Path, xtream_cluster: Option, mapping: &[(XtreamCluster, u32, u32)]) -> Result { - if let Some((cluster, cluster_start, _end)) = match xtream_cluster { - Some(clus) => mapping.iter().find(|(c, _, _)| *c == clus), - None => mapping.iter().find(|(_cluster, start, _end)| stream_id >= *start), - } { - let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, *cluster); - if stream_id >= *cluster_start { - return read_indexed_item::(&xtream_path, &idx_path, IndexRecord::get_index_offset(stream_id - cluster_start)); - } +fn xtream_read_item_for_stream_id(cfg: &Config, stream_id: u32, storage_path: &Path, cluster: &XtreamCluster) -> Result { + let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster); + { + let _file_lock = cfg.file_locks.read_lock(&xtream_path)?; + IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id) } - Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}"))) } -pub(crate) fn xtream_get_item_for_stream_id(stream_id: u32, config: &Config, target: &ConfigTarget, xtream_cluster: Option) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target.name.replace(' ', "_").as_str()) { - if let Some(mapping) = load_stream_id_cluster_mapping(&storage_path) { - if let Some(max) = mapping.iter().map(|(_cluster, _start, end)| end).max().copied() { - if max < stream_id { - // episoden id's fangen bei (max cluster id + 1) an. - // episode mapping har 3 u32 also 12 bytes - let index = stream_id - (max + 1); - if let Ok((_episode_id, provider_id, series_id)) = xtream_read_episode_id_mapping(&storage_path, index) { - if let Ok(mut pli) = _xtream_get_item_for_stream_id(series_id, &storage_path, xtream_cluster, &mapping) { - pli.provider_id = provider_id; - return Ok(pli); - } +fn xtream_read_series_item_for_stream_id(cfg: &Config, stream_id: u32, storage_path: &Path) -> Result { + let (xtream_path, idx_path) = xtream_get_file_paths_for_series(storage_path); + { + let _file_lock = cfg.file_locks.read_lock(&xtream_path)?; + return IndexedDocumentReader::::read_indexed_item(&xtream_path, &idx_path, stream_id); + } +} + +macro_rules! try_cluster { + ($xtream_cluster:expr, $item_type:expr, $virtual_id:expr) => { + $xtream_cluster.or_else(|| XtreamCluster::try_from($item_type).ok()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {}", $virtual_id))) + }; +} + +pub(crate) fn xtream_get_item_for_stream_id( + virtual_id: u32, + config: &Config, + target: &ConfigTarget, + xtream_cluster: Option, +) -> Result { + let target_path = get_target_storage_path(config, target.name.as_str()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name)))?; + let storage_path = xtream_get_storage_path(config, target.name.as_str()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {} xtream output", &target.name)))?; + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.read_lock(&target_id_mapping_file) + .map_err(|err| Error::new(ErrorKind::Other, format!("Could not get lock for id mapping for target {} err:{}", target.name, err.to_string())))?; + + let mut target_id_mapping = BPlusTreeQuery::::try_new(&target_id_mapping_file) + .map_err(|err| Error::new(ErrorKind::Other, format!("Could not load id mapping for target {} err:{}", target.name, err.to_string())))?; + + let mapping = target_id_mapping + .query(&virtual_id) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find mapping for target {} and id {}", target.name, virtual_id)))?; + + match mapping.item_type { + PlaylistItemType::SeriesInfo => xtream_read_series_item_for_stream_id(config, virtual_id, &storage_path), + PlaylistItemType::SeriesEpisode => { + let mut item = xtream_read_series_item_for_stream_id(config, mapping.parent_virtual_id, &storage_path)?; + item.provider_id = mapping.provider_id; + Ok(item) + } + PlaylistItemType::Catchup => { + let cluster = try_cluster!(xtream_cluster, mapping.item_type, virtual_id)?; + let mut item = xtream_read_item_for_stream_id(config, mapping.parent_virtual_id, &storage_path, &cluster)?; + item.provider_id = mapping.provider_id; + Ok(item) + } + _ => { + let cluster = try_cluster!(xtream_cluster, mapping.item_type, virtual_id)?; + xtream_read_item_for_stream_id(config, virtual_id, &storage_path, &cluster) + } + } + } +} + + +pub(crate) fn xtream_load_rewrite_playlist(cluster: &XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result { + if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) { + let (xtream_path, _) = xtream_get_file_paths(&storage_path, cluster); + { + let _file_lock = config.file_locks.read_lock(&xtream_path)?; + match IndexedDocumentReader::::new(&xtream_path) { + Ok(mut reader) => { + let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); + let result: Vec = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id) + .map(|pli| pli.to_doc(&options)).collect(); + if reader.by_ref().has_error() { + error!("Could not deserialize item {}", &xtream_path.to_str().unwrap()); + } else { + return Ok(serde_json::to_string(&result).unwrap()); } - return Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}"))); } - } - return _xtream_get_item_for_stream_id(stream_id, &storage_path, xtream_cluster, &mapping); - } - } - Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}"))) -} - -pub(crate) fn xtream_load_rewrite_playlist(cluster: XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target.name.replace(' ', "_").as_str()) { - let (xtream_path, idx_path) = xtream_get_file_paths(&storage_path, cluster); - match IndexedDocumentReader::::new(&xtream_path, &idx_path) { - Ok(mut reader) => { - let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); - let result: Vec = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id) - .map(|pli| pli.to_doc(&options)).collect(); - if reader.by_ref().has_error() { - error!("Could not deserialize item {}", &xtream_path.to_str().unwrap()); - } else { - return Ok(serde_json::to_string(&result).unwrap()); + Err(err) => { + error!("Could not deserialize file {} - {}", &xtream_path.to_str().unwrap(), err); } } - Err(err) => { - error!("Could not deserialize file {} - {}", &xtream_path.to_str().unwrap(), err); - } } } Err(Error::new(ErrorKind::Other, format!("Failed to find xtream storage for target {}", &target.name))) } - -// The mapping file record is episode_id 4bytes, episode_provider_id 4bytes, series_id 4bytes -fn read_last_id_from_episode_id_mapping_file(path: &Path) -> Result { - let mut file = File::open(path)?; - file.seek(SeekFrom::End(-12))?; - let mut buffer = [0; 4]; - file.read_exact(&mut buffer)?; - Ok(u32::from_le_bytes(buffer)) +macro_rules! try_option_ok { + ($option:expr) => { + match $option { + Some(value) => value, + None => return Ok(()), + } + }; } -pub(crate) fn xtream_get_max_series_info_episode_id(config: &Config, target_name: &str) -> Option { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let mapping_file_path = xtream_get_series_episode_id_mapping_file_path(&storage_path); - if mapping_file_path.exists() { - if let Ok(last_id) = read_last_id_from_episode_id_mapping_file(&mapping_file_path) { - return Some(last_id); +pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, + series_info_id: u32, + content: &str) -> Result<(), Error> { + let target_path = try_option_ok!(get_target_storage_path(config, target_name)); + let storage_path = try_option_ok!(xtream_get_storage_path(config, target_name)); + let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)); + + { + let _file_lock = config.file_locks.write_lock(&info_path)?; + let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?; + + writer + .write_doc(series_info_id, content) + .map_err(|_| Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}")))?; + + writer.flush()?; + } + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.write_lock(&target_id_mapping_file)?; + if let Ok(mut target_id_mapping) = BPlusTreeUpdate::::try_new(&target_id_mapping_file) { + if let Some(record) = target_id_mapping.query(&series_info_id) { + let new_record = record.copy_update_timestamp(); + let _ = target_id_mapping.update(&series_info_id, new_record); + } + }; + } + + Ok(()) +} + +// Reads the series info entry if exists, otherwise error +pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Option { + let target_path = get_target_storage_path(config, target_name)?; + let storage_path = xtream_get_storage_path(config, target_name)?; + + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.read_lock(&target_id_mapping_file).map_err(|err| { + error!("Could not lock id mapping for target {target_name}: {}", err); + Error::new(ErrorKind::Other, format!("ID mapping load error for target {target_name}")) + }).ok()?; + let mut target_id_mapping = BPlusTreeQuery::::try_new(&target_id_mapping_file) + .map_err(|err| { + error!("Could not load id mapping for target {target_name}: {}", err); + Error::new(ErrorKind::Other, format!("ID mapping load error for target {target_name}")) + }).ok()?; + + if let Some(id_record) = target_id_mapping.query(&series_id) { + if id_record.is_expired() { + return None; } } + } - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some(mapping) = load_stream_id_cluster_mapping(&storage_path) { - return mapping.iter().map(|(_, _, end)| end).max().copied(); - } + let (info_path, idx_path) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)?; + + if info_path.exists() && idx_path.exists() { + { + let _file_lock = config.file_locks.read_lock(&info_path).map_err(|err| { + error!("Could not lock document {:?}: {}", info_path, err); + Error::new(ErrorKind::Other, format!("Document Reader error for target {target_name}")) + }).ok()?; + return match IndexedDocumentReader::::read_indexed_item(&info_path, &idx_path, series_id) { + Ok(content) => Some(content), + Err(err) => { + error!("Failed to read series info for id {series_id} for {target_name}: {}", err); + None + } + }; } } None } -fn xtream_write_episode_id_mapping(storage_path: &Path, series_id: u32, episode_id_mapping: &[(u32, u32)]) -> Result<(), Error> { - let file_path = xtream_get_series_episode_id_mapping_file_path(storage_path); - if let Ok(mut file) = if file_path.exists() { - std::fs::OpenOptions::new() - .append(true) // Open in append mode - .open(file_path) - } else { - File::create(file_path) - } { - let stream_id_bytes: [u8; 4] = series_id.to_le_bytes(); - let mut bytes: Vec = Vec::new(); - for (episode_id, provider_id) in episode_id_mapping { - bytes.extend_from_slice(&episode_id.to_le_bytes()); - bytes.extend_from_slice(&provider_id.to_le_bytes()); - bytes.extend_from_slice(&stream_id_bytes); - } - return file_utils::check_write(&file.write_all(&bytes[..])); - } - Err(Error::new(ErrorKind::Other, format!("Failed to open series info idmapping file inside {}", storage_path.to_str().unwrap()))) -} +pub(crate) fn write_and_get_xtream_series_info( + config: &Config, + target: &ConfigTarget, + pli_series_info: &XtreamPlaylistItem, + content: &str, +) -> Result { + let mut doc = serde_json::from_str::(content) + .map_err(|_| Error::new(ErrorKind::Other, "Failed to parse JSON content"))?; -fn xtream_read_episode_id_mapping(storage_path: &Path, index: u32) -> Result<(u32, u32, u32), Error> { - let offset = index * 12; - let episode_file = xtream_get_series_episode_id_mapping_file_path(storage_path); - if episode_file.exists() { - if let Ok(mut file) = File::open(episode_file) { - file.seek(SeekFrom::Start(u64::from(offset)))?; - let mut bytes = [0u8; 4]; - file.read_exact(&mut bytes)?; - let episode_id = u32::from_le_bytes(bytes); - file.read_exact(&mut bytes)?; - let provider_id = u32::from_le_bytes(bytes); - file.read_exact(&mut bytes)?; - let series_id = u32::from_le_bytes(bytes); - Ok((episode_id, provider_id, series_id)) - } else { - Err(Error::new(ErrorKind::Other, format!("Could not find episode mapping at offset {offset}"))) - } - } else { - Err(Error::new(ErrorKind::Other, format!("Episode mapping not found at {}", storage_path.to_str().unwrap_or("")))) - } -} + let target_path = get_target_storage_path(config, target.name.as_str()) + .ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {}", target.name)))?; -pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str, - series_id: u32, episode_id_mapping: &[(u32, u32)], - content: &str) -> Result<(), Error> { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { - return match IndexedDocumentWriter::new_append(info_path.clone(), idx_path) { - Ok(mut writer) => { - match writer.write_doc(content) { - Ok((_, index_offset)) => { - let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path); - IndexRecord::to_file(&series_id_index_mapping_path, series_id, index_offset, true)?; - } - Err(_) => return Err(Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}"))) - } - xtream_write_episode_id_mapping(&storage_path, series_id, episode_id_mapping) + let episodes = doc.get_mut("episodes") + .and_then(Value::as_object_mut) + .ok_or_else(|| Error::new(ErrorKind::Other, "No episodes found in content"))?; + + { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let _file_lock = config.file_locks.write_lock(&target_id_mapping_file) + .map_err(|err| Error::new(ErrorKind::Other, format!("Could not load id mapping for target {} err:{}", target.name, err.to_string())))?; + let mut target_id_mapping = TargetIdMapping::new(&target_id_mapping_file); + let options = XtreamMappingOptions::from_target_options(target.options.as_ref()); + + for episode_list in episodes.values_mut().filter_map(Value::as_array_mut) { + for episode in episode_list.iter_mut().filter_map(Value::as_object_mut) { + if let Some(provider_id) = episode.get("id").and_then(Value::as_str).and_then(|id| id.parse::().ok()) { + let uuid = hash_string(&format!("{}/{}", pli_series_info.url, provider_id)); + let virtual_id = target_id_mapping.insert_entry(uuid, provider_id, &PlaylistItemType::SeriesEpisode, pli_series_info.virtual_id); + episode.insert("id".to_string(), Value::String(virtual_id.to_string())); } - Err(err) => Err(err) - }; - } - } - Ok(()) -} - -pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Result { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path); - if series_id_index_mapping_path.exists() { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) { - if info_path.exists() && idx_path.exists() { - let mut file = File::open(series_id_index_mapping_path)?; - let mut buffer = [0u8; 8]; - loop { - match file.read_exact(&mut buffer) { - Ok(()) => { - let stream_id = u32::from_le_bytes(buffer[..4].try_into().unwrap()); - if stream_id == series_id { - let index = u32::from_le_bytes(buffer[4..].try_into().unwrap()); - return read_indexed_item::(&info_path, &idx_path, index); - } - } - Err(e) => return Err(e), - } - } - } + if options.skip_series_direct_source { + episode.insert("direct_source".to_string(), Value::String(String::new())); } } } - } - Err(Error::new(ErrorKind::Other, format!("Failed to read series info for id {series_id} for {target_name}"))) -} -pub(crate) fn xtream_write_catchup_id_mapping(config: &Config, target_name: &str, id_mappings: &Vec<(u32, u32)>) -> Result<(), Error> { - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let file_path = xtream_get_catchup_id_mapping_file_path(&storage_path); - if let Ok(mut file) = if file_path.exists() { - std::fs::OpenOptions::new() - .append(true) // Open in append mode - .open(file_path) - } else { - File::create(file_path) - } { - for (provider_id, stream_id) in id_mappings { - let mut bytes: Vec = Vec::new(); - bytes.extend_from_slice(&provider_id.to_le_bytes()); - bytes.extend_from_slice(&stream_id.to_le_bytes()); - file_utils::check_write(&file.write_all(&bytes[..]))?; - } - return Ok(()); - } + drop(target_id_mapping); } - Err(Error::new(ErrorKind::Other, format!("Failed to get catchup info for {target_name}"))) -} + let result = serde_json::to_string(&doc) + .map_err(|_| Error::new(ErrorKind::Other, "Failed to serialize updated series info"))?; + xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result).ok(); -// Returns hashmap with provider_id -> new_stream_id -pub(crate) fn xtream_load_catchup_id_mapping(config: &Config, target_name: &str) -> HashMap { - let mut result = HashMap::new(); - if let Some(storage_path) = xtream_get_storage_path(config, target_name) { - let catchup_file = xtream_get_catchup_id_mapping_file_path(&storage_path); - if catchup_file.exists() { - if let Ok(file) = File::open(catchup_file) { - let mut reader = BufReader::new(&file); - let mut bytes = [0u8; 4]; - loop { - if reader.read_exact(&mut bytes).is_ok() { - let provider_id = u32::from_le_bytes(bytes); - let mut bytes = [0u8; 4]; - if reader.read_exact(&mut bytes).is_ok() { - let stream_id = u32::from_le_bytes(bytes); - result.insert(provider_id, stream_id); - } else { - break; - } - } else { - break; - } - } - } - } - } else { - error!("Failed to open catchup id-mapping file for {target_name}"); - } - result + Ok(result) } \ No newline at end of file diff --git a/src/test.rs b/src/test.rs index 4cd838d24..5c71cdd15 100644 --- a/src/test.rs +++ b/src/test.rs @@ -4,32 +4,14 @@ mod tests { use std::rc::Rc; use regex::Regex; use crate::filter::{get_filter, MockValueProcessor, ValueProvider}; - use crate::model::playlist::{PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; + use crate::model::playlist::{PlaylistItem, PlaylistItemHeader}; fn create_mock_pli(name: &str, group: &str) -> PlaylistItem { PlaylistItem { header: RefCell::new(PlaylistItemHeader { - uuid: Rc::new("".to_string()), - stream_id: Rc::new("".to_string()), - id: Rc::new("".to_string()), name: Rc::new(name.to_string()), - chno: Rc::new("".to_string()), - logo: Rc::new("".to_string()), - logo_small: Rc::new("".to_string()), group: Rc::new(group.to_string()), - title: Rc::new("".to_string()), - parent_code: Rc::new("".to_string()), - audio_track: Rc::new("".to_string()), - time_shift: Rc::new("".to_string()), - rec: Rc::new("".to_string()), - url: Rc::new("".to_string()), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Live, - additional_properties: None, - item_type: PlaylistItemType::Live, - series_fetched: false, - category_id: 0, - input_id: 0, + ..Default::default() }) } } diff --git a/src/utils/default_utils.rs b/src/utils/default_utils.rs index 758436c94..e51dbec7f 100644 --- a/src/utils/default_utils.rs +++ b/src/utils/default_utils.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::rc::Rc; use crate::model::config::ProcessingOrder; -use crate::model::playlist::{PlaylistItemType, XtreamCluster}; pub(crate) fn default_as_true() -> bool { true } @@ -23,8 +22,6 @@ pub(crate) fn default_as_empty_list() -> Vec { vec![] } pub(crate) fn default_as_two_u16() -> u16 { 2 } -pub(crate) fn default_playlist_item_type() -> PlaylistItemType { PlaylistItemType::Live } pub(crate) fn default_as_zero_u32() -> u32 { 0 } pub(crate) fn default_as_zero_u16() -> u16 { 0 } -pub(crate) fn default_stream_cluster() -> XtreamCluster { XtreamCluster::Live } diff --git a/src/utils/download.rs b/src/utils/download.rs index 9a0bf2358..a5ddff2de 100644 --- a/src/utils/download.rs +++ b/src/utils/download.rs @@ -30,8 +30,7 @@ pub(crate) async fn get_m3u_playlist(cfg: &Config, input: &ConfigInput, working_ let persist_file_path = prepare_file_path(input.persist.as_ref(), working_dir, ""); match request_utils::get_input_text_content(input, working_dir, &url, persist_file_path).await { Ok(text) => { - let lines = text.lines().map(String::from).collect::>(); - (m3u_parser::parse_m3u(cfg, input, &lines), vec![]) + (m3u_parser::parse_m3u(cfg, input, text.lines()), vec![]) } Err(err) => (vec![], vec![err]) } @@ -57,7 +56,7 @@ pub(crate) async fn get_xtream_playlist_series<'a>(fpl: &mut FetchedPlaylist<'a> match parse_xtream_series_info(&series_content, pli.header.borrow().group.as_str(), input) { Ok(series_info) => { if let Some(mut series) = series_info { - series.drain(..).for_each(|item| group_series.push(item)); + group_series.extend(series.drain(..)); } } Err(err) => errors.push(err), @@ -108,7 +107,7 @@ const ACTIONS: [(XtreamCluster, &str, &str); 3] = [ (XtreamCluster::Series, "get_series_categories", "get_series")]; pub(crate) async fn get_xtream_playlist(input: &ConfigInput, working_dir: &String) -> (Vec, Vec) { - let mut playlist: Vec = Vec::new(); + let mut playlist_groups: Vec = Vec::new(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); let base_url = format!("{}/player_api.php?username={}&password={}", input.url, username, password); @@ -123,35 +122,37 @@ pub(crate) async fn get_xtream_playlist(input: &ConfigInput, working_dir: &Strin let category_file_path = prepare_file_path(input.persist.as_ref(), working_dir, format!("{category}_").as_str()); let stream_file_path = prepare_file_path(input.persist.as_ref(), working_dir, format!("{stream}_").as_str()); - match request_utils::get_input_json_content(input, category_url.as_str(), category_file_path).await { - Ok(category_content) => { - match request_utils::get_input_json_content(input, stream_url.as_str(), stream_file_path).await { - Ok(stream_content) => { - match xtream_parser::parse_xtream(input, - *xtream_cluster, - &category_content, - &stream_content) { - Ok(sub_playlist_opt) => { - if let Some(mut sub_playlist) = sub_playlist_opt { - sub_playlist.drain(..).for_each(|group| playlist.push(group)); - } - } - Err(err) => errors.push(err) + match futures::join!( + request_utils::get_input_json_content(input, category_url.as_str(), category_file_path), + request_utils::get_input_json_content(input, stream_url.as_str(), stream_file_path) + ) { + (Ok(category_content), Ok(stream_content)) => { + match xtream_parser::parse_xtream(input, + *xtream_cluster, + &category_content, + &stream_content) { + Ok(sub_playlist_parsed) => { + if let Some(mut xtream_sub_playlist) = sub_playlist_parsed { + playlist_groups.extend(xtream_sub_playlist.drain(..)); } } Err(err) => errors.push(err) } - } - Err(err) => errors.push(err) + }, + (Err(err1), Err(err2)) => { + errors.extend([err1, err2]); + }, + (Err(err), _) => errors.push(err), + (_, Err(err)) => errors.push(err), } } } - playlist.sort_by(|a, b| a.title.partial_cmp(&b.title).unwrap_or(Ordering::Greater)); + playlist_groups.sort_by(|a, b| a.title.partial_cmp(&b.title).unwrap_or(Ordering::Greater)); - for (grp_id, plg) in (1_u32..).zip(playlist.iter_mut()) { + for (grp_id, plg) in (1_u32..).zip(playlist_groups.iter_mut()) { plg.id = grp_id; } - (playlist, errors) + (playlist_groups, errors) } pub(crate) async fn get_xmltv(_cfg: &Config, input: &ConfigInput, working_dir: &String) -> (Option, Vec) { diff --git a/src/utils/file_lock_manager.rs b/src/utils/file_lock_manager.rs new file mode 100644 index 000000000..6e16675e3 --- /dev/null +++ b/src/utils/file_lock_manager.rs @@ -0,0 +1,106 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::{fmt, io}; +use std::path::{Path, PathBuf}; + +#[derive(Clone)] +pub(crate) struct FileLockManager { + locks: Arc>>>>, +} + +impl FileLockManager { + pub(crate) fn new() -> Self { + Self { + locks: Arc::new(Mutex::new(HashMap::new())), + } + } + + // Acquires a read lock for the specified file and returns a FileReadGuard. + pub(crate) fn read_lock(&self, path: &Path) -> io::Result { + let file_lock = self.get_or_create_lock(path)?; + let guard = file_lock.read().map_err(|_| { + io::Error::new(io::ErrorKind::Other, "Failed to acquire read lock") + })?; + // Clone the Arc to avoid moving `file_lock` out, as it is still borrowed by `guard` + Ok(FileReadGuard::new(Arc::clone(&file_lock), guard)) + } + + // Acquires a write lock for the specified file and returns a FileWriteGuard. + pub(crate) fn write_lock(&self, path: &Path) -> io::Result { + let file_lock = self.get_or_create_lock(path)?; + let guard = file_lock.write().map_err(|_| { + io::Error::new(io::ErrorKind::Other, "Failed to acquire write lock") + })?; + // Clone the Arc to avoid moving `file_lock` out, as it is still borrowed by `guard` + Ok(FileWriteGuard::new(Arc::clone(&file_lock), guard)) + } + + // Helper function: retrieves or creates a lock for a file. + fn get_or_create_lock(&self, path: &Path) -> io::Result>> { + let mut locks = self.locks.lock().map_err(|_| { + io::Error::new(io::ErrorKind::Other, "Failed to acquire lock on lock manager") + })?; + + if let Some(lock) = locks.get(path) { + return Ok(lock.clone()); + } + + let file_lock = Arc::new(RwLock::new(())); + locks.insert(path.to_path_buf(), file_lock.clone()); + Ok(file_lock) + } +} + +impl Default for FileLockManager { + fn default() -> Self { + FileLockManager::new() + } +} + +impl fmt::Debug for FileLockManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Acquire the lock to safely access the HashMap + let locks = self.locks.lock().unwrap(); + // Format the paths in the HashMap for debug purposes + let keys: Vec<_> = locks.keys().collect(); + f.debug_struct("FileLockManager") + .field("locks", &keys) + .finish() + } +} + +// Define FileReadGuard to hold both the lock reference and the actual read guard. +#[allow(dead_code)] +pub(crate) struct FileReadGuard { + lock: Arc>, + guard: RwLockReadGuard<'static, ()>, +} + +impl FileReadGuard { + pub fn new(lock: Arc>, guard: RwLockReadGuard<'_, ()>) -> Self { + // Convert the lifetime of `guard` to 'static by transmuting. + let static_guard: RwLockReadGuard<'static, ()> = unsafe { std::mem::transmute(guard) }; + FileReadGuard { + lock, + guard: static_guard, + } + } +} + +// Define FileWriteGuard to hold both the lock reference and the actual write guard. +#[allow(dead_code)] +pub(crate) struct FileWriteGuard { + lock: Arc>, + guard: RwLockWriteGuard<'static, ()>, +} + +impl FileWriteGuard { + pub fn new(lock: Arc>, guard: RwLockWriteGuard<'_, ()>) -> Self { + // Convert the lifetime of `guard` to 'static by transmuting. + let static_guard: RwLockWriteGuard<'static, ()> = unsafe { std::mem::transmute(guard) }; + FileWriteGuard { + lock, + guard: static_guard, + } + } +} diff --git a/src/utils/file_utils.rs b/src/utils/file_utils.rs index 21b7a84c1..d1e785445 100644 --- a/src/utils/file_utils.rs +++ b/src/utils/file_utils.rs @@ -196,16 +196,3 @@ pub(crate) fn open_file_append(path: &Path, append: bool) -> Result Result<(File, File), std::io::Error> { - match open_file_append(path1, append) { - Ok(file1) => { - match open_file_append(path2, append) { - Ok(file2) => Ok((file1, file2)), - Err(err) => - Err(std::io::Error::new(err.kind(), format!("failed to create file {} - {}", path2.to_str().unwrap(), err))) - } - } - Err(err) => - Err(std::io::Error::new(err.kind(), format!("failed to create file {} - {}", path1.to_str().unwrap(), err))) - } -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 43590d89b..be648a6f0 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,4 +6,4 @@ pub (crate) mod json_utils; pub (crate) mod config_reader; pub (crate) mod default_utils; pub (crate) mod multi_file_reader; -// pub (crate) mod file_reader; +pub (crate) mod file_lock_manager; \ No newline at end of file diff --git a/src/utils/request_utils.rs b/src/utils/request_utils.rs index 10b282c86..9d13c5c1a 100644 --- a/src/utils/request_utils.rs +++ b/src/utils/request_utils.rs @@ -232,15 +232,15 @@ pub(crate) async fn get_input_json_content(input: &ConfigInput, url: &str, persi Err(e) => create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "cant download input url: {url} => {}", e) } } - -pub(crate) fn get_base_url(url: &str) -> Option { - if let Some((scheme_end, rest)) = url.split_once("://") { - let scheme = scheme_end; - if let Some(authority_end) = rest.find('/') { - let authority = &rest[..authority_end]; - return Some(format!("{}://{}", scheme, authority)); - } - return Some(format!("{}://{}", scheme, rest)); - } - None -} +// +// pub(crate) fn get_base_url(url: &str) -> Option { +// if let Some((scheme_end, rest)) = url.split_once("://") { +// let scheme = scheme_end; +// if let Some(authority_end) = rest.find('/') { +// let authority = &rest[..authority_end]; +// return Some(format!("{}://{}", scheme, authority)); +// } +// return Some(format!("{}://{}", scheme, rest)); +// } +// None +// } diff --git a/test/rest-api.http b/test/rest-api.http index 01c45fe29..e7a59df00 100644 --- a/test/rest-api.http +++ b/test/rest-api.http @@ -17,10 +17,19 @@ Content-Type: application/json {"username": "test", "password": "test"} ### xtream live_categories -GET http://localhost:8901/m3u?username=me&password=me +GET http://localhost:8901/player_api.php?action=get_live_categories&username=xt&password=xt ### xtream live_streams -GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&password=xt&category_id=203 +GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&password=xt&category_id=102 ### xtream series_categories GET http://localhost:8901/player_api.php?action=get_series_categories&username=xt&password=xt + +### xtream series +GET http://localhost:8901/player_api.php?action=get_series&username=xt&password=xt&category_id=120 + +### xtream series info +GET http://localhost:8901/player_api.php?action=get_series_info&username=xt&password=xt&series_id=16441 + +### xtream series stream + GET http://localhost:8901/series/xt/xt/18130 \ No newline at end of file