euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-09 17:32:16 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

virtual id handling

Browse Source
This commit is contained in:
euzu
2024-10-29 17:51:40 +01:00
parent abd4d3a335 02994fc906
commit 0e30f82a68
37 changed files with 1893 additions and 1217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3
View File
@@ -1,4 +1,7 @@
# Changelog
# 2.0.6 (2024-10-*)
- breaking change virtual_id handling. You need to clear the data directory.
# 2.0.5(2024-10-16)
- input url supports now scheme `file://...` (which is not necessary because file paths are supported). Gzip files are also supported.
- sort takes now a sequence for channel values which has higher priority than sort order
Cargo.lock
Generated
+40 -44
View File
@@ -506,9 +506,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.7.2"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da"
[[package]]
name = "bytestring"
@@ -527,9 +527,9 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6"
[[package]]
name = "cc"
version = "1.1.30"
version = "1.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945"
checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f"
dependencies = [
"jobserver",
"libc",
@@ -760,9 +760,9 @@ dependencies = [
[[package]]
name = "encoding_rs"
version = "0.8.34"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
@@ -1259,9 +1259,9 @@ dependencies = [
[[package]]
name = "impl-more"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e658178c10c747241199382079c0f195ce229866fbf4aa0d46fa6107fe33d2ec"
checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0"
[[package]]
name = "indexmap"
@@ -1370,9 +1370,9 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388"
[[package]]
name = "libc"
version = "0.2.159"
version = "0.2.161"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
[[package]]
name = "libnghttp2-sys"
@@ -1445,7 +1445,6 @@ dependencies = [
"actix-server",
"actix-web",
"actix-web-httpauth",
"base64 0.22.1",
"bincode",
"blake3",
"chrono",
@@ -1607,9 +1606,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
[[package]]
name = "openssl"
version = "0.10.67"
version = "0.10.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b8cefcf97f41316955f9294cd61f639bdcfa9f2f230faac6cb896aa8ab64704"
checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5"
dependencies = [
"bitflags 2.6.0",
"cfg-if",
@@ -1639,9 +1638,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-src"
version = "300.3.2+3.3.2"
version = "300.4.0+3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b"
checksum = "a709e02f2b4aca747929cca5ed248880847c650233cf8b8cdc48f40aaf4898a6"
dependencies = [
"cc",
]
@@ -1785,18 +1784,18 @@ dependencies = [
[[package]]
name = "pin-project"
version = "1.1.6"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec"
checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.6"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8"
checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c"
dependencies = [
"proc-macro2",
"quote",
@@ -1805,9 +1804,9 @@ dependencies = [
[[package]]
name = "pin-project-lite"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
[[package]]
name = "pin-utils"
@@ -1854,9 +1853,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.87"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a"
checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
dependencies = [
"unicode-ident",
]
@@ -2149,9 +2148,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.14"
version = "0.23.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8"
checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993"
dependencies = [
"once_cell",
"ring",
@@ -2239,18 +2238,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "serde"
version = "1.0.210"
version = "1.0.213"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.210"
version = "1.0.213"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5"
dependencies = [
"proc-macro2",
"quote",
@@ -2259,9 +2258,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.128"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
dependencies = [
"itoa",
"memchr",
@@ -2405,9 +2404,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.79"
version = "2.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56"
dependencies = [
"proc-macro2",
"quote",
@@ -2459,18 +2458,18 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.64"
version = "1.0.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.64"
version = "1.0.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602"
dependencies = [
"proc-macro2",
"quote",
@@ -2525,9 +2524,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.40.0"
version = "1.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb"
dependencies = [
"backtrace",
"bytes",
@@ -2642,12 +2641,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "unicase"
version = "2.7.0"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
dependencies = [
"version_check",
]
checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df"
[[package]]
name = "unicode-bidi"
Cargo.toml
+3 -4
View File
@@ -14,10 +14,10 @@ strip = true
[dependencies]
serde = { version = "1.0", features = ["derive", "rc"] }
serde_yaml = "0.9.33"
serde_yaml = "0.9.34"
serde_json = "1"
quick-xml = { version = "0.36", features = ["serialize"] }
regex = "1.10"
regex = "1.11"
clap = { version = "4", features = ["derive"] }
url = "2.5"
reqwest = { version = "0", features = ["blocking", "json", "stream", "rustls-tls"] }
@@ -49,5 +49,4 @@ rand = "0.8"
rpassword = "7.3"
flate2 = "1"
time = "0.3"
blake3 = "1.5.4"
base64 = "0.22.1"
blake3 = "1.5"
bin/print_idx_file.py
-41
View File
@@ -1,41 +0,0 @@
#!/usr/bin/env python3
"""
This script reads the index files for xtream file indexes
Each index entry consist of 8 bytes where the first 4 bytes and the last 4 bytes are unsigned 32-bit integers (u32).
@see IndexRecord
The IndexRecord can be use for different purposes, f.e.
- id -> id mapping
- id -> index
- index -> size (in this case the offset for the id is calculated through the "cluster_id_start->cluster mapping" offset shift)
"""
import sys
import struct
def print_u32_values(file_path):
try:
with open(file_path, "rb") as file:
while True:
chunk = file.read(8)
if not chunk:
break
if len(chunk) == 8:
first_u32, last_u32 = struct.unpack("<II", chunk)
print(f"{first_u32} : {last_u32}")
else:
print("Incomplete chunk:", chunk)
except FileNotFoundError:
print(f"Error: File '{file_path}' not found")
if __name__ == "__main__":
if len(sys.argv) != 2:
script_name = os.path.basename(sys.argv[0])
print("Usage: {script_name} <file_name>")
sys.exit(1)
file_path = sys.argv[1]
print_u32_values(file_path)
src/api/m3u_api.rs
+15 -8
View File
@@ -5,6 +5,7 @@ use crate::api::api_utils::{get_user_target, get_user_target_by_credentials, str
use crate::api::api_model::{AppState, UserApiRequest};
use crate::model::config::TargetType;
use crate::repository::m3u_repository::{m3u_get_file_paths, m3u_get_item_for_stream_id, m3u_load_rewrite_playlist};
use crate::repository::storage::get_target_storage_path;
async fn m3u_api(
api_req: web::Query<UserApiRequest>,
@@ -15,7 +16,6 @@ async fn m3u_api(
//let api_req = UserApiRequest::from_map(&_api_req);
match get_user_target(&api_req, &app_state) {
Some((user, target)) => {
// let filename = target.get_m3u_filename();
if let Some(content) = m3u_load_rewrite_playlist(&app_state.config, target, &user) {
HttpResponse::Ok().content_type(mime::TEXT_PLAIN_UTF_8).body(content)
} else {
@@ -38,15 +38,22 @@ async fn m3u_api_stream(
if let Ok(m3u_stream_id) = stream_id.parse::<u32>() {
if let Some((_user, target)) = get_user_target_by_credentials(&username, &password, &api_req, &app_state) {
if target.has_output(&TargetType::M3u) {
if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&app_state.config, target) {
match m3u_get_item_for_stream_id(m3u_stream_id, &m3u_path, &idx_path) {
Ok(m3u_item) => {
return stream_response(m3u_item.url.as_str(), &req, None).await;
}
Err(err) => {
error!("Failed to get m3u url: {}", err);
match get_target_storage_path(&app_state.config, target.name.as_str()) {
Some(target_path) => {
if let Some((m3u_path, idx_path)) = m3u_get_file_paths(&target_path) {
match m3u_get_item_for_stream_id(&app_state.config, m3u_stream_id, &m3u_path, &idx_path) {
Ok(m3u_item) => {
return stream_response(m3u_item.url.as_str(), &req, None).await;
}
Err(err) => {
error!("Failed to get m3u url: {}", err);
}
}
}
}
None => {
error!("Failed to get target path for {}", target.name);
}
}
}
}
src/api/mod.rs
+1 -1
View File
@@ -7,4 +7,4 @@ mod xtream_api;
mod m3u_api;
mod xmltv_api;
mod scheduler;
mod web_index;
mod web_index;
src/api/scheduler.rs
+60 -8
View File
@@ -1,29 +1,81 @@
use std::str::FromStr;
use std::time::Duration;
use std::time::{Duration, Instant, SystemTime};
use actix_web::web::Data;
use chrono::Local;
use chrono::{DateTime, FixedOffset, Local};
use cron::Schedule;
use log::error;
use crate::api::api_model::AppState;
use crate::exit;
use crate::processing::playlist_processor::exec_processing;
fn datetime_to_instant(datetime: DateTime<FixedOffset>) -> Instant {
// Convert DateTime<FixedOffset> to SystemTime
let target_system_time: SystemTime = datetime.into();
// Get the current SystemTime
let now_system_time = SystemTime::now();
// Calculate the duration between now and the target time
let duration_until = target_system_time
.duration_since(now_system_time)
.unwrap_or_else(|_| Duration::from_secs(0));
// Get the current Instant and add the duration to calculate the target Instant
Instant::now() + duration_until
}
pub(crate) async fn start_scheduler(expression: &str, data: Data<AppState>) -> ! {
match Schedule::from_str(expression) {
Ok(schedule) => {
let offset = *Local::now().offset();
loop {
let mut upcoming = schedule.upcoming(offset).take(1);
actix_rt::time::sleep(Duration::from_millis(500)).await;
let local = &Local::now();
if let Some(datetime) = upcoming.next() {
if datetime.timestamp() <= local.timestamp() {
exec_processing(data.config.clone(), data.targets.clone()).await;
}
actix_rt::time::sleep_until(actix_rt::time::Instant::from(datetime_to_instant(datetime))).await;
exec_processing(data.config.clone(), data.targets.clone()).await;
}
}
}
Err(err) => exit!("Failed to start scheduler: {}", err)
}
}
#[cfg(test)]
mod tests {
use std::str::FromStr;
use std::sync::atomic::{AtomicU8, Ordering};
use chrono::Local;
use cron::Schedule;
use crate::api::scheduler::datetime_to_instant;
#[actix_rt::test]
async fn test_run_scheduler() {
// Define a cron expression that runs every second
let expression = "0/1 * * * * * *"; // every second
let runs = AtomicU8::new(0);
let run_me = || runs.fetch_add(1, Ordering::Relaxed);
let start = std::time::Instant::now();
match Schedule::from_str(expression) {
Ok(schedule) => {
let offset = *Local::now().offset();
loop {
let mut upcoming = schedule.upcoming(offset).take(1);
if let Some(datetime) = upcoming.next() {
actix_rt::time::sleep_until(actix_rt::time::Instant::from(datetime_to_instant(datetime))).await;
run_me();
}
if runs.load(Ordering::Relaxed) == 6 {
break;
}
}
}
Err(_) => {}
};
let duration = start.elapsed();
assert!(runs.load(Ordering::Relaxed) == 6, "Failed to run");
assert!(duration.as_secs() > 4, "Failed time");
}
}
src/api/xmltv_api.rs
+5 -1
View File
@@ -17,6 +17,7 @@ use crate::model::api_proxy::{ProxyType, ProxyUserCredentials};
use crate::model::config::{Config, ConfigInput, ConfigTarget};
use crate::model::config::TargetType;
use crate::repository::m3u_repository::m3u_get_epg_file_path;
use crate::repository::storage::get_target_storage_path;
use crate::repository::xtream_repository::{xtream_get_epg_file_path, xtream_get_storage_path};
use crate::utils::{file_utils, request_utils};
@@ -51,10 +52,13 @@ fn get_epg_path_for_target_of_type(target_name: &str, file_path: Option<PathBuf>
}
fn get_epg_path_for_target(config: &Config, target: &ConfigTarget) -> Option<PathBuf> {
// TODO if we share the same virtual_id for epg, can we store an epg file for the target ?
for output in &target.output {
match output.target {
TargetType::M3u => {
return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path(config, target));
if let Some(target_path) = get_target_storage_path(config, &target.name) {
return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path(&target_path));
}
}
TargetType::Xtream => {
if let Some(storage_path) = xtream_get_storage_path(config, &target.name) {
src/api/xtream_api.rs
+102 -148
View File
@@ -1,7 +1,6 @@
// https://github.com/tellytv/go.xtream-codes/blob/master/structs.go
use std::collections::HashMap;
use std::convert::TryFrom;
use std::fmt::{Display, Formatter};
use std::io::{Error, ErrorKind};
use std::path::Path;
@@ -17,12 +16,49 @@ use crate::api::api_utils::{get_user_server_info, get_user_target, get_user_targ
use crate::model::api_proxy::{ProxyType, ProxyUserCredentials};
use crate::model::config::{Config, ConfigInput, ConfigTarget};
use crate::model::config::TargetType;
use crate::model::playlist::{XtreamCluster, XtreamPlaylistItem};
use crate::model::playlist::{PlaylistItemType, XtreamCluster, XtreamPlaylistItem};
use crate::model::xtream::XtreamMappingOptions;
use crate::repository::storage::{get_target_storage_path, hash_string};
use crate::repository::target_id_mapping::TargetIdMapping;
use crate::repository::xtream_repository;
use crate::utils::{json_utils, request_utils};
macro_rules! try_option_bad_request {
($option:expr, $msg_is_error:expr, $msg:expr) => {
match $option {
Some(value) => value,
None => {
if $msg_is_error {error!("{}", $msg);} else {debug!("{}", $msg);}
return HttpResponse::BadRequest().finish();
}
}
};
($option:expr) => {
match $option {
Some(value) => value,
None => return HttpResponse::BadRequest().finish(),
}
};
}
macro_rules! try_result_bad_request {
($option:expr, $msg_is_error:expr, $msg:expr) => {
match $option {
Ok(value) => value,
Err(_) => {
if $msg_is_error {error!("{}", $msg);} else {debug!("{}", $msg);}
return HttpResponse::BadRequest().finish();
}
}
};
($option:expr) => {
match $option {
Ok(value) => value,
Err(_) => return HttpResponse::BadRequest().finish(),
}
};
}
enum XtreamApiStreamContext {
LiveAlt,
Live,
@@ -109,7 +145,6 @@ fn get_xtream_player_api_stream_url(input: &ConfigInput, context: &str, action_p
}
}
fn get_user_info(user: &ProxyUserCredentials, cfg: &Config) -> XtreamAuthorizationResponse {
let server_info = get_user_server_info(cfg, user);
@@ -157,43 +192,34 @@ async fn xtream_player_api_stream(
app_state: &web::Data<AppState>,
stream_req: XtreamApiStreamRequest<'_>,
) -> HttpResponse {
if let Some((user, target)) = get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state) {
let target_name = &target.name;
if target.has_output(&TargetType::Xtream) {
let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id);
let req_stream_id: u32 = match FromStr::from_str(action_stream_id.trim()) {
Ok(id) => id,
Err(_) => return HttpResponse::BadRequest().finish()
};
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, None) {
let input_id: u16 = pli.input_id;
if let Some(input) = app_state.config.get_input_by_id(input_id) {
let mut query_path = if stream_req.action_path.is_empty() { String::new() } else { format!("{}/", stream_req.action_path) };
query_path = format!("{query_path}{}{stream_ext}", pli.provider_id);
if let Some(stream_url) = get_xtream_player_api_stream_url(input, stream_req.context.to_string().as_str(), query_path.as_str()) {
if user.proxy == ProxyType::Redirect {
debug!("Redirecting stream request to {stream_url}");
return HttpResponse::Found().insert_header(("Location", stream_url)).finish();
}
return stream_response(&stream_url, req, Some(input)).await;
}
error!("Cant find stream url for target {target_name}, context {}, stream_id {req_stream_id}", stream_req.context);
} else {
error!("Cant find input for target {target_name}, context {}, stream_id {req_stream_id}", stream_req.context);
}
} else {
error!("Failed to read xtream item for stream id {}", req_stream_id);
}
} else {
debug!("Target has no xtream output {}", target_name);
}
} else {
debug!("Could not find any user {}", stream_req.username);
let (user, target) = try_option_bad_request!(get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state), false, format!("Could not find any user {}", stream_req.username));
let target_name = &target.name;
if !target.has_output(&TargetType::Xtream) {
debug!("Target has no xtream output {}", target_name);
return HttpResponse::BadRequest().finish();
}
HttpResponse::BadRequest().finish()
let (action_stream_id, stream_ext) = xtream_api_request_separate_number_and_rest(stream_req.stream_id);
let virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse());
let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None), true, format!("Failed to read xtream item for stream id {}", virtual_id));
let input = try_option_bad_request!(app_state.config.get_input_by_id(pli.input_id), true, format!("Cant find input for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context));
let query_path = if stream_req.action_path.is_empty() {
format!("{}{stream_ext}", pli.provider_id)
} else {
format!("{}/{}{stream_ext}", stream_req.action_path, pli.provider_id)
};
let stream_url = try_option_bad_request!(get_xtream_player_api_stream_url(input, stream_req.context.to_string().as_str(), &query_path), true, format!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", stream_req.context));
if user.proxy == ProxyType::Redirect {
debug!("Redirecting stream request to {stream_url}");
return HttpResponse::Found().insert_header(("Location", stream_url)).finish();
}
stream_response(&stream_url, req, Some(input)).await
}
async fn xtream_player_api_live_stream(
req: HttpRequest,
api_req: web::Query<UserApiRequest>,
@@ -263,7 +289,7 @@ async fn xtream_player_api_streaming_timeshift(
fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result<String, Error> {
if let Ok(mut doc) = serde_json::from_str::<Map<String, Value>>(content) {
if let Some(Value::Object(movie_data)) = doc.get_mut("movie_data") {
let stream_id = pli.stream_id;
let stream_id = pli.virtual_id;
let category_id = pli.category_id;
movie_data.insert("stream_id".to_string(), Value::Number(serde_json::value::Number::from(stream_id)));
movie_data.insert("category_id".to_string(), Value::Number(serde_json::value::Number::from(category_id)));
@@ -278,44 +304,7 @@ fn get_xtream_vod_info(target: &ConfigTarget, pli: &XtreamPlaylistItem, content:
}
}
}
Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.stream_id)))
}
fn get_xtream_series_info(config: &Config, target: &ConfigTarget, pli: &XtreamPlaylistItem, content: &str) -> Result<String, Error> {
if let Ok(mut doc) = serde_json::from_str::<Value>(content) {
let mut new_id_to_provider_id_mapping: Vec<(u32, u32)> = Vec::new();
if let Some(mut new_id) = xtream_repository::xtream_get_max_series_info_episode_id(config, target.name.replace(' ', "_").as_str()) {
if let Some(episodes) = doc.get_mut("episodes") {
if let Some(episodes_map) = episodes.as_object_mut() {
let options = XtreamMappingOptions::from_target_options(target.options.as_ref());
for (_season, episode_list) in episodes_map {
// Iterate over items in the episode
if let Some(entries) = episode_list.as_array_mut() {
for entry in entries {
if let Some(episode) = entry.as_object_mut() {
if let Some(episode_id) = episode.get("id") {
if let Ok(provider_id) = episode_id.as_str().unwrap().parse::<u32>() {
new_id += 1;
new_id_to_provider_id_mapping.push((new_id, provider_id));
episode.insert("id".to_string(), Value::String(new_id.to_string()));
}
}
if options.skip_series_direct_source {
episode.insert("direct_source".to_string(), Value::String(String::new()));
}
}
}
}
}
}
if let Ok(result) = serde_json::to_string(&doc) {
let _ = xtream_repository::xtream_write_series_info(config, target.name.replace(' ', "_").as_str(), pli.stream_id, &new_id_to_provider_id_mapping, &result);
return Ok(result);
}
}
}
}
Err(Error::new(ErrorKind::Other, format!("Failed to get series info for id {}", pli.stream_id)))
Err(Error::new(ErrorKind::Other, format!("Failed to get vod info for id {}", pli.virtual_id)))
}
async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) -> Result<String, Error> {
@@ -325,7 +314,7 @@ async fn xtream_get_stream_info_content(info_url: &str, input: &ConfigInput) ->
async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &ConfigTarget,
pli: &XtreamPlaylistItem, info_url: &str, cluster: XtreamCluster) -> Result<String, Error> {
if cluster == XtreamCluster::Series {
if let Ok(content) = xtream_repository::xtream_load_series_info(config, target.name.replace(' ', "_").as_str(), pli.stream_id) {
if let Some(content) = xtream_repository::xtream_load_series_info(config, target.name.as_str(), pli.virtual_id) {
return Ok(content);
}
}
@@ -333,28 +322,24 @@ async fn xtream_get_stream_info(config: &Config, input: &ConfigInput, target: &C
if let Ok(content) = xtream_get_stream_info_content(info_url, input).await {
return match cluster {
XtreamCluster::Live => Ok(content),
XtreamCluster::Video => {
return get_xtream_vod_info(target, pli, &content);
}
XtreamCluster::Series => {
return get_xtream_series_info(config, target, pli, &content);
}
XtreamCluster::Video => get_xtream_vod_info(target, pli, &content),
XtreamCluster::Series => xtream_repository::write_and_get_xtream_series_info(config, target, pli, &content),
};
}
Err(Error::new(std::io::ErrorKind::Other, format!("Cant find stream with id: {}/{}/{}",
target.name.replace(' ', "_").as_str(), &cluster, pli.stream_id)))
target.name.replace(' ', "_").as_str(), &cluster, pli.virtual_id)))
}
async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserCredentials,
target: &ConfigTarget, stream_id: &str,
cluster: XtreamCluster) -> HttpResponse {
let req_stream_id: u32 = match FromStr::from_str(stream_id) {
let virtual_id: u32 = match FromStr::from_str(stream_id) {
Ok(id) => id,
Err(_) => return HttpResponse::BadRequest().finish()
};
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, Some(cluster)) {
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(cluster)) {
let input_id = pli.input_id;
if let Some(input) = app_state.config.get_input_by_id(input_id) {
if let Some(info_url) = get_xtream_player_api_info_url(input, cluster, pli.provider_id) {
@@ -377,12 +362,12 @@ async fn xtream_get_stream_info_response(app_state: &AppState, user: &ProxyUserC
async fn xtream_get_short_epg(app_state: &AppState, user: &ProxyUserCredentials, target: &ConfigTarget, stream_id: &str, limit: &str) -> HttpResponse {
let target_name = &target.name;
if target.has_output(&TargetType::Xtream) {
let req_stream_id: u32 = match FromStr::from_str(stream_id.trim()) {
let virtual_id: u32 = match FromStr::from_str(stream_id.trim()) {
Ok(id) => id,
Err(_) => return HttpResponse::BadRequest().finish()
};
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, None) {
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, None) {
let input_id: u16 = pli.input_id;
if let Some(input) = app_state.config.get_input_by_id(input_id) {
if let Some(action_url) = get_xtream_player_api_action_url(input, "get_short_epg") {
@@ -431,59 +416,33 @@ async fn xtream_player_api_handle_content_action(config: &Config, target_name: &
}
async fn xtream_get_catchup_response(app_state: &AppState, target: &ConfigTarget, stream_id: &str, start: &str, end: &str) -> HttpResponse {
let req_stream_id: u32 = match FromStr::from_str(stream_id) {
Ok(id) => id,
Err(_) => return HttpResponse::BadRequest().finish()
};
let virtual_id: u32 = try_result_bad_request!(FromStr::from_str(stream_id));
let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(virtual_id, &app_state.config, target, Some(XtreamCluster::Live)));
let input = try_option_bad_request!(app_state.config.get_input_by_id(pli.input_id));
let info_url = try_option_bad_request!(get_xtream_player_api_action_url(input, "get_simple_data_table").map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id)));
let content = try_result_bad_request!(xtream_get_stream_info_content(info_url.as_str(), input).await);
let mut doc: Map<String, Value> = try_result_bad_request!(serde_json::from_str(&content));
let epg_listings = try_option_bad_request!(doc.get_mut("epg_listings").and_then(Value::as_array_mut));
let target_path = try_option_bad_request!(get_target_storage_path(&app_state.config, target.name.as_str()));
let mut target_id_mapping = TargetIdMapping::new(&target_path);
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(req_stream_id, &app_state.config, target, Some(XtreamCluster::Live)) {
let input_id = pli.input_id;
if let Some(input) = app_state.config.get_input_by_id(input_id) {
if let Some(info_url) = get_xtream_player_api_action_url(input, "get_simple_data_table")
.map(|action_url| format!("{action_url}&stream_id={}&start={start}&end={end}", pli.provider_id)) {
if let Ok(content) = xtream_get_stream_info_content(info_url.as_str(), input).await {
if let Ok(mut doc) = serde_json::from_str::<Map<String, Value>>(content.as_str()) {
if let Some(epg_listings) = doc.get_mut("epg_listings") {
if let Some(epg_listing_list) = epg_listings.as_array_mut() {
let mapping = xtream_repository::xtream_load_catchup_id_mapping(&app_state.config, target.name.as_str());
let mut max_id = u32::try_from(mapping.len()).unwrap();
let mut new_id_mappings = Vec::new();
for epg_list_value in epg_listing_list {
if let Some(epg_list_item) = epg_list_value.as_object_mut() {
// TODO epg_id
if let Some(Some(provider_id)) = epg_list_item.get("id").map(|v| v.as_str()) {
if let Ok(provider_stream_id) = &FromStr::from_str(provider_id) {
let stream_id = match mapping.get(provider_stream_id) {
None => {
max_id += 1;
new_id_mappings.push((*provider_stream_id, max_id));
max_id
}
Some(mapped_id) => *mapped_id
};
epg_list_item.insert("id".to_string(), Value::String(stream_id.to_string()));
}
}
}
}
if !new_id_mappings.is_empty() {
if let Err(err) = xtream_repository::xtream_write_catchup_id_mapping(&app_state.config, target.name.as_str(), &new_id_mappings) {
error!("Failed to write catchup id mapping {err}");
return HttpResponse::BadRequest().finish();
}
}
}
}
if let Ok(result) = serde_json::to_string(&doc) {
return HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(result);
}
}
}
}
for epg_list_item in epg_listings.iter_mut().filter_map(Value::as_object_mut) {
// TODO epg_id
if let Some(catchup_provider_id) = epg_list_item.get("id").and_then(Value::as_str).and_then(|id| id.parse::<u32>().ok()) {
let uuid = hash_string(&format!("{}/{}", pli.url, catchup_provider_id));
let virtual_id = target_id_mapping.insert_entry(uuid, catchup_provider_id, &PlaylistItemType::Catchup, pli.provider_id);
epg_list_item.insert("id".to_string(), Value::String(virtual_id.to_string()));
}
}
HttpResponse::BadRequest().finish()
if let Err(err) = target_id_mapping.persist() {
error!("Failed to write catchup id mapping {err}");
return HttpResponse::BadRequest().finish();
}
match serde_json::to_string(&doc) {
Ok(result) => HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(result),
Err(_) => HttpResponse::BadRequest().finish(),
}
}
async fn xtream_player_api(
@@ -530,9 +489,9 @@ async fn xtream_player_api(
} else {
let cat_id = if category_id.is_empty() { 0 } else { category_id.parse::<u32>().unwrap_or(0) };
match match action {
"get_live_streams" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Live, &app_state.config, target, cat_id),
"get_vod_streams" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Video, &app_state.config, target, cat_id),
"get_series" => xtream_repository::xtream_load_rewrite_playlist(XtreamCluster::Series, &app_state.config, target, cat_id),
"get_live_streams" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Live, &app_state.config, target, cat_id),
"get_vod_streams" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Video, &app_state.config, target, cat_id),
"get_series" => xtream_repository::xtream_load_rewrite_playlist(&XtreamCluster::Series, &app_state.config, target, cat_id),
_ => Err(Error::new(ErrorKind::Unsupported, format!("Cant find action: {action} for target: {target_name}"))),
} {
Ok(payload) => HttpResponse::Ok().content_type(mime::APPLICATION_JSON).body(payload),
@@ -549,13 +508,8 @@ async fn xtream_player_api(
}
}
_ => {
if api_req.action.is_empty() {
debug!("Paremeter action is empty!");
HttpResponse::Unauthorized().finish()
} else {
debug!("cant find user!");
HttpResponse::BadRequest().finish()
}
debug!("{}", if api_req.action.is_empty() { "Paremeter action is empty!" } else { "cant find user!" });
HttpResponse::BadRequest().finish()
}
}
}
src/filter.rs
+5 -4
View File
@@ -194,9 +194,10 @@ impl std::fmt::Display for Filter {
Filter::TypeComparison(field, item_type) => {
write!(f, "{} = {}", field, match item_type {
PlaylistItemType::Live => "live",
PlaylistItemType::Movie => "vod",
PlaylistItemType::Video => "vod",
PlaylistItemType::Series => "series",
PlaylistItemType::SeriesInfo => "series" // yes series-info is handled as series in filter
PlaylistItemType::SeriesInfo => "series", // yes series-info is handled as series in filter
_ => "unsupported"
})
}
Filter::Group(stmt) => {
@@ -268,8 +269,8 @@ fn get_parser_field_comparison(expr: Pair<Rule>, templates: &Vec<PatternTemplate
fn get_filter_item_type(text_item_type: &str) -> Option<PlaylistItemType> {
if text_item_type.eq_ignore_ascii_case("live") {
Some(PlaylistItemType::Live)
} else if text_item_type.eq_ignore_ascii_case("movie") || text_item_type.eq_ignore_ascii_case("vod") {
Some(PlaylistItemType::Movie)
} else if text_item_type.eq_ignore_ascii_case("movie") || text_item_type.eq_ignore_ascii_case("video") || text_item_type.eq_ignore_ascii_case("vod") {
Some(PlaylistItemType::Video)
} else if text_item_type.eq_ignore_ascii_case("series") {
Some(PlaylistItemType::Series)
} else if text_item_type.eq_ignore_ascii_case("series-info") {
src/main.rs
+4 -5
View File
@@ -147,11 +147,10 @@ fn init_logger(log_level: &str) {
let mut log_builder = Builder::from_default_env();
if log_level.contains('=') {
let pairs: Vec<&str> = log_level.split(',').collect();
for pair in pairs {
let kv: Vec<&str> = pair.split('=').collect();
if kv.len() == 2 {
log_builder.filter_module(kv[0].trim(), get_log_level(kv[1].trim()));
for pair in log_level.split(',').filter(|s| s.contains('=')) {
let mut kv_iter = pair.split('=').map(str::trim);
if let (Some(module), Some(level)) = (kv_iter.next(), kv_iter.next()) {
log_builder.filter_module(module, get_log_level(level));
}
}
} else {
src/model/config.rs
+10 -1
View File
@@ -24,6 +24,7 @@ use crate::utils::{config_reader, file_utils};
use crate::utils::default_utils::{default_as_default, default_as_false, default_as_true,
default_as_empty_list, default_as_frm, default_as_empty_map,
default_as_zero_u8, default_as_two_u16};
use crate::utils::file_lock_manager::FileLockManager;
pub(crate) const MAPPER_ATTRIBUTE_FIELDS: &[&str] = &[
"name", "title", "group", "id", "chno", "logo",
@@ -42,6 +43,13 @@ macro_rules! valid_property {
}};
}
#[macro_export]
macro_rules! create_m3u_filter_error {
($kind: expr, $($arg:tt)*) => {
M3uFilterError::new($kind, format!($($arg)*))
}
}
#[macro_export]
macro_rules! create_m3u_filter_error_result {
($kind: expr, $($arg:tt)*) => {
@@ -802,7 +810,8 @@ pub(crate) struct Config {
pub t_sources_file_path: String,
#[serde(skip_serializing, skip_deserializing)]
pub t_api_proxy_file_path: String,
#[serde(skip)]
pub file_locks : Arc<FileLockManager>
}
impl Config {
src/model/playlist.rs
+93 -46
View File
@@ -2,18 +2,17 @@ use std::cell::RefCell;
use std::cmp::PartialEq;
use std::fmt::{Display, Formatter};
use std::rc::Rc;
use base64::Engine;
use blake3::Hasher;
use base64::engine::general_purpose;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::model::config::{ConfigInput, ConfigTarget};
use crate::model::xmltv::TVGuide;
use crate::model::xtream::{xtream_playlistitem_to_document, XtreamMappingOptions};
use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32, default_playlist_item_type, default_stream_cluster};
use crate::utils::request_utils::get_base_url;
use crate::processing::m3u_parser::extract_id_from_url;
use crate::repository::storage::hash_string;
use crate::utils::default_utils::{default_as_false, default_as_zero_u16, default_as_zero_u32};
// https://de.wikipedia.org/wiki/M3U
// https://siptv.eu/howto/playlist.html
@@ -44,31 +43,77 @@ pub(crate) enum XtreamCluster {
Series = 3,
}
impl Display for XtreamCluster {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", match self {
XtreamCluster::Live => "live",
XtreamCluster::Video => "movie",
XtreamCluster::Series => "series",
})
impl XtreamCluster {
pub fn as_str(&self) -> &str {
match self {
XtreamCluster::Live => "Live",
XtreamCluster::Video => "Video",
XtreamCluster::Series => "Series",
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
impl Default for XtreamCluster {
fn default() -> Self {
XtreamCluster::Live
}
}
impl Display for XtreamCluster {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.as_str())
}
}
impl TryFrom<PlaylistItemType> for XtreamCluster {
type Error = String;
fn try_from(item_type: PlaylistItemType) -> Result<Self, Self::Error> {
match item_type {
PlaylistItemType::Live => Ok(XtreamCluster::Live),
PlaylistItemType::Video => Ok(XtreamCluster::Video),
PlaylistItemType::Series => Ok(XtreamCluster::Series),
_ => Err(format!("Cant convert {item_type}")),
}
}
}
#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
#[repr(u8)]
pub(crate) enum PlaylistItemType {
Live = 1,
Movie = 2,
Series = 3,
SeriesInfo = 4,
Video = 2,
Series = 3, // xtream series description
SeriesInfo = 4, // xtream series info fetched for series description
SeriesEpisode = 5, // from SeriesInfo parsed episodes
Catchup = 6,
}
impl Default for PlaylistItemType {
fn default() -> Self {
PlaylistItemType::Live
}
}
impl From<XtreamCluster> for PlaylistItemType {
fn from(xtream_cluster: XtreamCluster) -> Self {
match xtream_cluster {
XtreamCluster::Live => PlaylistItemType::Live,
XtreamCluster::Video => PlaylistItemType::Video,
XtreamCluster::Series => PlaylistItemType::SeriesInfo,
}
}
}
impl Display for PlaylistItemType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", match self {
PlaylistItemType::Live => "live",
PlaylistItemType::Movie => "movie",
PlaylistItemType::Video => "video",
PlaylistItemType::Series => "series",
PlaylistItemType::SeriesInfo => "series-info",
PlaylistItemType::SeriesEpisode => "series-episode",
PlaylistItemType::Catchup => "catchup"
})
}
}
@@ -78,11 +123,11 @@ pub(crate) trait FieldAccessor {
fn set_field(&mut self, field: &str, value: &str) -> bool;
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub(crate) struct PlaylistItemHeader {
pub uuid: Rc<String>, // calculated
pub stream_id: Rc<String>, // virtual id
pub uuid: Rc<[u8; 32]>, // calculated
pub id: Rc<String>, // provider id
pub virtual_id: u32, // virtual id
pub name: Rc<String>,
pub chno: Rc<String>,
pub logo: Rc<String>,
@@ -95,10 +140,9 @@ pub(crate) struct PlaylistItemHeader {
pub rec: Rc<String>,
pub url: Rc<String>,
pub epg_channel_id: Option<Rc<String>>,
#[serde(default = "default_stream_cluster")]
pub xtream_cluster: XtreamCluster,
pub additional_properties: Option<Value>,
#[serde(default = "default_playlist_item_type", skip_serializing, skip_deserializing)]
#[serde(skip_serializing, skip_deserializing)]
pub item_type: PlaylistItemType,
#[serde(default = "default_as_false", skip_serializing, skip_deserializing)]
pub series_fetched: bool, // only used for series_info
@@ -109,22 +153,27 @@ pub(crate) struct PlaylistItemHeader {
}
impl PlaylistItemHeader {
pub(crate) fn gen_uuid(&mut self) {
//let base_url = get_base_url(&self.url).unwrap_or_else(|| self.url.to_string());
// Create a Blake3 hasher
let mut hasher = Hasher::new();
// the url should be different for each entry.
hasher.update(self.url.as_bytes());
// Finalize and get the hash result
let hash_bytes = hasher.finalize();
// Encode the reduced 128-bit hash to Base62
let mut encoded_key = String::new();
general_purpose::STANDARD.encode_string(&hash_bytes.as_bytes(), &mut encoded_key);
self.uuid = Rc::new(encoded_key);
self.uuid = Rc::new(hash_string(&self.url))
}
pub(crate) fn get_uuid(&self) -> &str {
self.uuid.as_ref()
pub(crate) fn get_uuid(&self) -> &Rc<[u8; 32]> {
&self.uuid
}
pub(crate) fn get_provider_id(&mut self) -> Option<u32> {
match self.id.parse::<u32>() {
Ok(id) => Some(id),
Err(_) => match extract_id_from_url(&self.url) {
Some(id) => match id.parse::<u32>() {
Ok(newid) => {
self.id = Rc::new(newid.to_string());
Some(newid)
}
Err(_) => None,
},
None => None,
}
}
}
}
@@ -172,11 +221,11 @@ macro_rules! generate_field_accessor_impl_for_playlist_item_header {
}
}
generate_field_accessor_impl_for_playlist_item_header!(id, stream_id, name, chno, logo, logo_small, group, title, parent_code, audio_track, time_shift, rec, url;);
generate_field_accessor_impl_for_playlist_item_header!(id, /*virtual_id,*/ name, chno, logo, logo_small, group, title, parent_code, audio_track, time_shift, rec, url;);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct M3uPlaylistItem {
pub stream_id: Rc<String>,
pub virtual_id: u32,
pub provider_id: Rc<String>,
pub name: Rc<String>,
pub chno: Rc<String>,
@@ -218,7 +267,7 @@ impl M3uPlaylistItem {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct XtreamPlaylistItem {
pub stream_id: u32,
pub virtual_id: u32,
pub provider_id: u32,
pub name: Rc<String>,
pub logo: Rc<String>,
@@ -252,7 +301,7 @@ impl PlaylistItem {
pub fn to_m3u(&self) -> M3uPlaylistItem {
let header = self.header.borrow();
M3uPlaylistItem {
stream_id: Rc::clone(&header.stream_id),
virtual_id: header.virtual_id,
provider_id: Rc::clone(&header.id),
name: Rc::clone(&header.name),
chno: Rc::clone(&header.chno),
@@ -275,7 +324,7 @@ impl PlaylistItem {
match header.id.parse::<u32>() {
Ok(provider_id) => {
Ok(XtreamPlaylistItem {
stream_id: header.stream_id.parse::<u32>().unwrap_or(0),
virtual_id: header.virtual_id,
provider_id,
name: Rc::clone(&header.name),
logo: Rc::clone(&header.logo),
@@ -312,14 +361,12 @@ pub(crate) struct PlaylistGroup {
pub id: u32,
pub title: Rc<String>,
pub channels: Vec<PlaylistItem>,
#[serde(default = "default_stream_cluster", skip_serializing, skip_deserializing)]
#[serde(skip_serializing, skip_deserializing)]
pub xtream_cluster: XtreamCluster,
}
impl PlaylistGroup {
pub(crate) fn on_load(&mut self) {
self.channels.iter().for_each(|pl| pl.header.borrow_mut().gen_uuid());
}
}
}
src/model/xtream.rs
+27 -16
View File
@@ -8,7 +8,7 @@ use serde_json::{Map, Value};
use crate::model::config::ConfigTargetOptions;
use crate::model::playlist::{PlaylistItem, XtreamCluster, XtreamPlaylistItem};
use crate::utils::default_utils::{default_as_empty_rc_str, default_as_empty_list};
use crate::utils::default_utils::{default_as_empty_list, default_as_empty_rc_str};
const LIVE_STREAM_FIELDS: &[&str] = &[];
@@ -28,8 +28,8 @@ const SERIES_STREAM_FIELDS: &[&str] = &[
fn deserialize_number_from_string<'de, D, T: DeserializeOwned>(
deserializer: D,
) -> Result<Option<T>, D::Error>
where
D: Deserializer<'de>,
where
D: Deserializer<'de>,
{
// we define a local enum type inside of the function
// because it is untagged, serde will deserialize as the first variant
@@ -73,8 +73,8 @@ fn value_to_string(v: &Value) -> Option<String> {
}
fn deserialize_as_option_rc_string<'de, D>(deserializer: D) -> Result<Option<Rc<String>>, D::Error>
where
D: Deserializer<'de>,
where
D: Deserializer<'de>,
{
let value: Value = Deserialize::deserialize(deserializer)?;
@@ -86,8 +86,8 @@ fn deserialize_as_option_rc_string<'de, D>(deserializer: D) -> Result<Option<Rc<
}
fn deserialize_as_rc_string<'de, D>(deserializer: D) -> Result<Rc<String>, D::Error>
where
D: Deserializer<'de>,
where
D: Deserializer<'de>,
{
let value: Value = Deserialize::deserialize(deserializer)?;
@@ -98,8 +98,8 @@ fn deserialize_as_rc_string<'de, D>(deserializer: D) -> Result<Rc<String>, D::Er
}
fn deserialize_as_string_array<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
where
D: Deserializer<'de>,
where
D: Deserializer<'de>,
{
Value::deserialize(deserializer).map(|v| match v {
Value::Array(value) => Some(value_to_string_array(&value)),
@@ -131,9 +131,9 @@ pub(crate) struct XtreamStream {
#[serde(default, deserialize_with = "deserialize_as_rc_string")]
pub category_id: Rc<String>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub stream_id: Option<i32>,
pub stream_id: Option<u32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub series_id: Option<i32>,
pub series_id: Option<u32>,
#[serde(default = "default_as_empty_rc_str", deserialize_with = "deserialize_as_rc_string")]
pub stream_icon: Rc<String>,
#[serde(default = "default_as_empty_rc_str", deserialize_with = "deserialize_as_rc_string")]
@@ -217,10 +217,9 @@ macro_rules! add_i64_property_if_exists {
}
}
impl XtreamStream {
pub(crate) fn get_stream_id(&self) -> String {
self.stream_id.map_or_else(|| self.series_id.map_or_else(String::new, |seid| format!("{seid}")), |sid| format!("{sid}"))
pub(crate) fn get_stream_id(&self) -> u32 {
self.stream_id.unwrap_or(self.series_id.unwrap_or(0))
}
pub(crate) fn get_additional_properties(&self) -> Option<Value> {
@@ -315,6 +314,18 @@ pub(crate) struct XtreamSeriesInfoEpisode {
pub direct_source: String,
}
// impl XtreamSeriesInfoEpisode {
// pub(crate) fn get_id(&self) -> u32 {
// match self.id.parse::<u32>() {
// Ok(id) => id,
// Err(_) => {
// error!("Failed to convert id to number {}", self.id);
// 0
// }
// }
// }
// }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct XtreamSeriesInfo {
pub seasons: Vec<XtreamSeriesInfoSeason>,
@@ -358,7 +369,7 @@ impl XtreamMappingOptions {
pub fn from_target_options(options: Option<&ConfigTargetOptions>) -> Self {
let (skip_live_direct_source, skip_video_direct_source, skip_series_direct_source) = options
.map_or((false, false, false), |o| (o.xtream_skip_live_direct_source,
o.xtream_skip_video_direct_source, o.xtream_skip_series_direct_source));
o.xtream_skip_video_direct_source, o.xtream_skip_series_direct_source));
Self {
skip_live_direct_source,
skip_video_direct_source,
@@ -414,7 +425,7 @@ fn append_prepared_series_properties(add_props: Option<&Map<String, Value>>, doc
}
pub(crate) fn xtream_playlistitem_to_document(pli: &XtreamPlaylistItem, options: &XtreamMappingOptions) -> serde_json::Value {
let stream_id_value = Value::Number(serde_json::Number::from(pli.stream_id));
let stream_id_value = Value::Number(serde_json::Number::from(pli.virtual_id));
let mut document = serde_json::Map::from_iter([
("category_id".to_string(), Value::String(format!("{}", &pli.category_id))),
("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(pli.category_id))]))),
src/processing/m3u_parser.rs
+21 -32
View File
@@ -4,7 +4,6 @@ use std::rc::Rc;
use crate::model::config::{Config, ConfigInput};
use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
use crate::utils::default_utils::{default_as_empty_rc_str, default_playlist_item_type, default_stream_cluster};
use crate::utils::string_utils;
#[inline]
@@ -76,27 +75,10 @@ fn skip_digit(it: &mut std::str::Chars) -> Option<char> {
fn create_empty_playlistitem_header(input_id: u16, url: &str) -> PlaylistItemHeader {
PlaylistItemHeader {
uuid: default_as_empty_rc_str(),
id: default_as_empty_rc_str(),
stream_id: default_as_empty_rc_str(),
name: default_as_empty_rc_str(),
chno: default_as_empty_rc_str(),
logo: default_as_empty_rc_str(),
logo_small: default_as_empty_rc_str(),
group: default_as_empty_rc_str(),
title: default_as_empty_rc_str(),
parent_code: default_as_empty_rc_str(),
audio_track: default_as_empty_rc_str(),
time_shift: default_as_empty_rc_str(),
rec: default_as_empty_rc_str(),
url: Rc::new(url.to_owned()),
epg_channel_id: None,
item_type: default_playlist_item_type(),
xtream_cluster: default_stream_cluster(),
additional_properties: None,
series_fetched: false,
category_id: 0,
input_id,
..Default::default()
}
}
@@ -147,14 +129,14 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str
plih.id = Rc::new(chanid);
}
}
plih.stream_id = Rc::clone(&plih.id);
// plih.virtual_id = plih.id;
plih.epg_channel_id = Some(Rc::clone(&plih.id));
}
if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) {
// TODO find Series based on group or configured names
plih.xtream_cluster = XtreamCluster::Video;
plih.item_type = PlaylistItemType::Movie;
plih.item_type = PlaylistItemType::Video;
}
{
@@ -174,7 +156,7 @@ fn process_header(input: &ConfigInput, video_suffixes: &Vec<&str>, content: &str
pub(crate) fn extract_id_from_url(url: &str) -> Option<String> {
if let Some(filename) = url.split('/').last() {
return if let Some(index) = filename.rfind('.') {
return if let Some(index) = filename.rfind('.') {
Some(filename[..index].to_string())
} else {
Some(filename.to_string())
@@ -183,14 +165,17 @@ pub(crate) fn extract_id_from_url(url: &str) -> Option<String> {
None
}
pub(crate) fn consume_m3u<F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: impl Iterator<Item=String>, mut visit: F) {
pub(crate) fn consume_m3u<'a, I, F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInput, lines: I, mut visit: F)
where
I: Iterator<Item=&'a str>,
{
let mut header: Option<String> = None;
let mut group: Option<String> = None;
let video_suffixes = cfg.video.as_ref().unwrap().extensions.iter().map(String::as_str).collect::<Vec<&str>>();
for line in lines {
if line.starts_with("#EXTINF") {
header = Some(line);
header = Some(String::from(line));
continue;
}
if line.starts_with("#EXTGRP") {
@@ -219,29 +204,33 @@ pub(crate) fn consume_m3u<F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigIn
}
}
pub(crate) fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: &[String]) -> Vec<PlaylistGroup> {
pub(crate) fn parse_m3u<'a, I>(cfg: &Config, input: &ConfigInput, lines: I) -> Vec<PlaylistGroup>
where
I: Iterator<Item=&'a str>,
{
let mut sort_order: Vec<Vec<PlaylistItem>> = vec![];
let mut idx: usize = 0;
let mut sort_order_idx: usize = 0;
let mut group_map: std::collections::HashMap<Rc<String>, usize> = std::collections::HashMap::new();
consume_m3u(cfg, input, lines.iter().cloned(), |item| {
consume_m3u(cfg, input, lines, |item| {
// keep the original sort order for groups and group the playlist items
let key = Rc::clone(&item.header.borrow().group);
match group_map.entry(key) {
std::collections::hash_map::Entry::Vacant(v) => {
v.insert(idx);
idx += 1;
v.insert(sort_order_idx);
sort_order.push(vec![item]);
sort_order_idx += 1;
}
std::collections::hash_map::Entry::Occupied(o) => {
sort_order.get_mut(*o.get()).unwrap().push(item);
}
}
});
let mut grp_id = 0;
let result: Vec<PlaylistGroup> = sort_order.drain(..).map(|channels| {
// create a group based on the first playlist item
let channel = channels.first();
let cluster = channel.map(|pli| pli.header.borrow().xtream_cluster).unwrap();
let group_title = channel.map(|pli| Rc::clone(&pli.header.borrow().group)).unwrap();
let (cluster, group_title) = channel.map(|pli|
(pli.header.borrow().xtream_cluster, Rc::clone(&pli.header.borrow().group))).unwrap();
grp_id += 1;
PlaylistGroup { id: grp_id, xtream_cluster: cluster, title: Rc::clone(&group_title), channels }
}).collect();
src/processing/playlist_processor.rs
+37 -35
View File
@@ -25,8 +25,8 @@ use crate::processing::playlist_watch::process_group_watch;
use crate::processing::xmltv_parser::flatten_tvguide;
use crate::processing::xtream_processor::playlist_resolve_series;
use crate::repository::playlist_repository::persist_playlist;
use crate::utils::download;
use crate::utils::default_utils::default_as_default;
use crate::utils::download;
fn is_valid(pli: &PlaylistItem, target: &ConfigTarget) -> bool {
let provider = ValueProvider { pli: RefCell::new(pli) };
@@ -96,7 +96,7 @@ fn playlistitem_comparator(a: &PlaylistItem, b: &PlaylistItem, channel_sort: &Co
}
}
}
},
}
None => {
let ordering = value_a.partial_cmp(&value_b).unwrap();
match channel_sort.order {
@@ -255,7 +255,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) {
for channel in &plg.channels {
let provider = ValueProvider { pli: RefCell::new(channel) };
if counter.filter.filter(&provider, &mut mock_processor) {
let new_value = if counter.modifier == CounterModifier::Assign {
let new_value = if counter.modifier == CounterModifier::Assign {
cntval.to_string()
} else {
let value = match channel.header.borrow_mut().get_field(&counter.field) {
@@ -264,7 +264,7 @@ fn map_playlist_counter(target: &ConfigTarget, playlist: &[PlaylistGroup]) {
};
if counter.modifier == CounterModifier::Suffix {
format!("{}{}{}", value, counter.concat, cntval.to_string())
} else {
} else {
format!("{}{}{}", cntval.to_string(), counter.concat, value)
}
};
@@ -295,10 +295,11 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<ProcessTargets>) -> (Vec<InputStats>, Vec<M3uFilterError>) {
let source = cfg.sources.get(source_idx).unwrap();
let mut source_playlists = Vec::new();
let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count();
let mut errors = vec![];
let mut stats = HashMap::<u16, InputStats>::new();
let mut source_playlists = Vec::new();
let enabled_inputs = source.inputs.iter().filter(|item| item.enabled).count();
// Downlod the sources
for input in &source.inputs {
let input_id = input.id;
if is_input_enabled(enabled_inputs, input.enabled, input_id, &user_targets) {
@@ -306,16 +307,16 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
InputType::M3u => download::get_m3u_playlist(&cfg, input, &cfg.working_dir).await,
InputType::Xtream => download::get_xtream_playlist(input, &cfg.working_dir).await,
};
// @TODO optmization dont hold tv_guide in memory, persist raw and later use sax parser to extract.
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
download::get_xmltv(&cfg, input, &cfg.working_dir).await
} else {
(None, vec![])
};
error_list.drain(..).for_each(|err| errors.push(err));
tvguide_errors.drain(..).for_each(|err| errors.push(err));
errors.extend(error_list.drain(..).chain(tvguide_errors.drain(..)));
let input_name = match &input.name {
Some(name_val) => name_val.as_str(),
None => input.url.as_str(),
Some(name_val) => name_val.as_str()
};
let group_count = playlistgroups.len();
let channel_count = playlistgroups.iter()
@@ -334,19 +335,8 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
}
);
}
stats.insert(input_id, InputStats {
name: input_name.to_string(),
input_type: input.input_type.clone(),
error_count: error_list.len(),
raw_stats: PlaylistStats {
group_count,
channel_count,
},
processed_stats: PlaylistStats {
group_count: 0,
channel_count: 0,
},
});
stats.insert(input_id, create_input_stat(group_count, channel_count, error_list.len(),
input.input_type.clone(), input_name));
}
}
if source_playlists.is_empty() {
@@ -362,7 +352,7 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
if is_target_enabled(target, &user_targets) {
match process_playlist(&mut source_playlists, target, &cfg, &mut stats, &mut errors).await {
Ok(()) => {}
Err(mut err) => err.drain(..).for_each(|e| errors.push(e))
Err(mut err) => errors.extend(err.drain(..))
}
}
}
@@ -370,6 +360,22 @@ async fn process_source(cfg: Arc<Config>, source_idx: usize, user_targets: Arc<P
(stats.drain().map(|(_, v)| v).collect(), errors)
}
fn create_input_stat(group_count: usize, channel_count: usize, error_count: usize, input_type: InputType, input_name: &str) -> InputStats {
InputStats {
name: input_name.to_string(),
input_type: input_type,
error_count: error_count,
raw_stats: PlaylistStats {
group_count,
channel_count,
},
processed_stats: PlaylistStats {
group_count: 0,
channel_count: 0,
},
}
}
async fn process_sources(config: Arc<Config>, user_targets: Arc<ProcessTargets>) -> (Vec<InputStats>, Vec<M3uFilterError>) {
let mut handle_list = vec![];
let thread_num = config.threads;
@@ -390,10 +396,8 @@ async fn process_sources(config: Arc<Config>, user_targets: Arc<ProcessTargets>)
let (mut res_stats, mut res_errors) = System::new().block_on(async {
process_source(cfg, index, usr_trgts).await
});
res_errors.drain(..)
.for_each(|err| shared_errors.lock().unwrap().push(err));
res_stats.drain(..)
.for_each(|stat| shared_stats.lock().unwrap().push(stat));
shared_errors.lock().unwrap().extend(res_errors.drain(..));
shared_stats.lock().unwrap().extend(res_stats.drain(..));
};
handles.push(thread::spawn(process));
if handles.len() >= thread_num as usize {
@@ -401,10 +405,8 @@ async fn process_sources(config: Arc<Config>, user_targets: Arc<ProcessTargets>)
}
} else {
let (mut res_stats, mut res_errors) = process_source(cfg, index, usr_trgts).await;
res_errors.drain(..)
.for_each(|err| shared_errors.lock().unwrap().push(err));
res_stats.drain(..)
.for_each(|stat| shared_stats.lock().unwrap().push(stat));
shared_errors.lock().unwrap().extend(res_errors.drain(..));
shared_stats.lock().unwrap().extend(res_stats.drain(..));
}
}
for handle in handle_list {
@@ -464,7 +466,8 @@ fn flatten_groups(mut playlistgroups: Vec<PlaylistGroup>) -> Vec<PlaylistGroup>
}
async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>],
target: &ConfigTarget, cfg: &Config,
target: &ConfigTarget,
cfg: &Config,
stats: &mut HashMap<u16, InputStats>,
errors: &mut Vec<M3uFilterError>) -> Result<(), Vec<M3uFilterError>> {
let pipe = get_processing_pipe(target);
@@ -493,12 +496,11 @@ async fn process_playlist<'a>(playlists: &mut [FetchedPlaylist<'a>],
let mut new_epg = vec![];
new_fetched_playlists.drain(..).for_each(|mut fp| {
// collect all epg_channel ids
let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels)
.filter_map(|c| c.header.borrow().epg_channel_id.clone()).collect();
fp.playlistgroups.drain(..).for_each(|group| {
new_playlist.push(group);
});
new_playlist.extend(fp.playlistgroups.drain(..));
if !epg_channel_ids.is_empty() {
if let Some(tv_guide) = fp.epg {
debug!("found epg information for {}", &target.name);
src/processing/xtream_parser.rs
+46 -59
View File
@@ -8,12 +8,11 @@ use crate::create_m3u_filter_error_result;
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::model::config::ConfigInput;
use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamStream};
use crate::utils::default_utils::default_as_empty_rc_str;
use crate::model::xtream::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream};
fn map_to_xtream_category(category: &Value) -> Result<Vec<XtreamCategory>, M3uFilterError> {
match serde_json::from_value::<Vec<XtreamCategory>>(category.to_owned()) {
Ok(category_list) => Ok(category_list),
fn map_to_xtream_category(categories: &Value) -> Result<Vec<XtreamCategory>, M3uFilterError> {
match serde_json::from_value::<Vec<XtreamCategory>>(categories.to_owned()) {
Ok(xtream_categories) => Ok(xtream_categories),
Err(err) => {
create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "Failed to process categories {}", &err)
}
@@ -29,6 +28,16 @@ fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: &Value) -> Resu
}
}
fn create_xtream_series_info_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Rc<String> {
if episode.direct_source.is_empty() {
let ext = episode.container_extension.clone();
let stream_base_url = format!("{url}/series/{username}/{password}/{}.{ext}", episode.id);
Rc::new(stream_base_url)
} else {
Rc::new(episode.direct_source.clone())
}
}
pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &ConfigInput) -> Result<Option<Vec<PlaylistItem>>, M3uFilterError> {
let url = input.url.as_str();
let username = input.username.as_ref().map_or("", |v| v);
@@ -39,33 +48,18 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &
let result: Vec<PlaylistItem> = series_info.episodes.values().flatten().map(|episode|
PlaylistItem {
header: RefCell::new(PlaylistItemHeader {
uuid: default_as_empty_rc_str(),
id: Rc::new(episode.id.clone()),
stream_id: Rc::new(episode.id.clone()),
id: Rc::new(episode.id.to_string()),
name: Rc::new(episode.title.clone()),
chno: default_as_empty_rc_str(),
logo: Rc::new(episode.info.movie_image.clone()),
logo_small: default_as_empty_rc_str(),
group: Rc::new(group_title.to_string()),
title: Rc::new(episode.title.clone()),
parent_code: default_as_empty_rc_str(),
audio_track: default_as_empty_rc_str(),
time_shift: default_as_empty_rc_str(),
rec: default_as_empty_rc_str(),
url: if episode.direct_source.is_empty() {
let ext = episode.container_extension.clone();
let stream_base_url = format!("{}/series/{}/{}/{}.{}", url, username, password, episode.id.as_str(), ext);
Rc::new(stream_base_url)
} else {
Rc::new(episode.direct_source.clone())
},
epg_channel_id: None,
url: create_xtream_series_info_url(url, username, password, episode),
item_type: PlaylistItemType::Series,
xtream_cluster: XtreamCluster::Series,
additional_properties: episode.get_additional_properties(&series_info),
series_fetched: false,
category_id: 0,
input_id: input.id,
..Default::default()
})
}).collect();
if result.is_empty() { Ok(None) } else { Ok(Some(result)) }
@@ -76,69 +70,61 @@ pub(crate) fn parse_xtream_series_info(info: &Value, group_title: &str, input: &
}
}
fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &str, password: &str, stream: &XtreamStream) -> Rc<String> {
if stream.direct_source.is_empty() {
let stream_base_url = match xtream_cluster {
XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()),
XtreamCluster::Video => {
let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str());
format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext)
}
XtreamCluster::Series =>
format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}",
url, username, password, &stream.get_stream_id())
};
Rc::new(stream_base_url)
} else {
Rc::clone(&stream.direct_source)
}
}
pub(crate) fn parse_xtream(input: &ConfigInput,
xtream_cluster: XtreamCluster,
category: &Value,
categories: &Value,
streams: &Value) -> Result<Option<Vec<PlaylistGroup>>, M3uFilterError> {
match map_to_xtream_category(category) {
Ok(mut categories) => {
match map_to_xtream_category(categories) {
Ok(mut xtream_categories) => {
let input_id = input.id;
let url = input.url.as_str();
let username = input.username.as_ref().map_or("", |v| v);
let password = input.password.as_ref().map_or("", |v| v);
return match map_to_xtream_streams(xtream_cluster, streams) {
Ok(streams) => {
Ok(xtream_streams) => {
let group_map: HashMap::<Rc<String>, RefCell<XtreamCategory>> =
categories.drain(..).map(|category|
xtream_categories.drain(..).map(|category|
(Rc::clone(&category.category_id), RefCell::new(category))
).collect();
for stream in streams {
for stream in xtream_streams {
if let Some(group) = group_map.get(&stream.category_id) {
let mut grp = group.borrow_mut();
let category_name = &grp.category_name;
let item = PlaylistItem {
header: RefCell::new(PlaylistItemHeader {
uuid: default_as_empty_rc_str(),
id: Rc::new(stream.get_stream_id()),
stream_id: Rc::new(stream.get_stream_id()),
id: Rc::new(stream.get_stream_id().to_string()),
name: Rc::clone(&stream.name),
chno: default_as_empty_rc_str(),
logo: Rc::clone(&stream.stream_icon),
logo_small: default_as_empty_rc_str(),
group: Rc::clone(category_name),
title: Rc::clone(&stream.name),
parent_code: default_as_empty_rc_str(),
audio_track: default_as_empty_rc_str(),
time_shift: default_as_empty_rc_str(),
rec: default_as_empty_rc_str(),
url: if stream.direct_source.is_empty() {
let stream_base_url = match xtream_cluster {
XtreamCluster::Live => format!("{}/live/{}/{}/{}.ts", url, username, password, &stream.get_stream_id()),
XtreamCluster::Video => {
let ext = stream.container_extension.as_ref().map_or("mp4", |e| e.as_str());
format!("{}/movie/{}/{}/{}.{}", url, username, password, &stream.get_stream_id(), ext)
}
XtreamCluster::Series =>
format!("{}/player_api.php?username={}&password={}&action=get_series_info&series_id={}",
url, username, password, &stream.get_stream_id())
};
Rc::new(stream_base_url)
} else {
Rc::clone(&stream.direct_source)
},
url: create_xtream_url(xtream_cluster, url, username, password, &stream),
epg_channel_id: stream.epg_channel_id.clone(),
item_type: match xtream_cluster {
XtreamCluster::Live => PlaylistItemType::Live,
XtreamCluster::Video => PlaylistItemType::Movie,
XtreamCluster::Series => PlaylistItemType::SeriesInfo,
},
item_type: PlaylistItemType::from(xtream_cluster),
xtream_cluster,
additional_properties: stream.get_additional_properties(),
series_fetched: false,
category_id: 0,
input_id,
..Default::default()
}),
};
grp.add(item);
@@ -160,4 +146,5 @@ pub(crate) fn parse_xtream(input: &ConfigInput,
}
Err(err) => Err(err)
}
}
}
src/repository/bplustree.rs
+532 -97
View File
@@ -1,25 +1,81 @@
use std::fs::File;
use std::io::{self};
use flate2::Compression;
use flate2::write::{GzEncoder};
use flate2::read::{GzDecoder};
use std::array::TryFromSliceError;
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::marker::PhantomData;
use std::path::Path;
use flate2::Compression;
use log::error;
use serde::{Deserialize, Serialize};
const T: usize = 3; // Minimum degree (T), meaning each node can contain at most 2*T - 1 keys
const BINCODE_OVERHEAD: usize = 4;
const BLOCK_SIZE: usize = 4096;
const POINTER_SIZE: usize = size_of::<Option<u64>>();
#[derive(Serialize, Deserialize, Debug, Clone)]
struct KeyValue<K, V> {
key: K,
value: V,
fn is_multiple_of_block_size(file: &File) -> io::Result<bool> {
let file_size = file.metadata()?.len(); // Get the file size in bytes
Ok(file_size % (BLOCK_SIZE as u64) == 0) // Check if file size is a multiple of BLOCK_SIZE
}
fn is_file_valid(file: File) -> io::Result<File> {
match is_multiple_of_block_size(&file) {
Ok(valid) => {
if !valid {
return Err(io::Error::new(io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}")));
}
}
Err(err) => return Err(err)
}
Ok(file)
}
#[inline]
fn u32_from_bytes(bytes: &[u8]) -> io::Result<u32> {
Ok(u32::from_le_bytes(bytes.try_into().map_err(|e: TryFromSliceError| io::Error::new(io::ErrorKind::Other, e.to_string()))?))
}
#[inline]
fn bincode_serialize<T: ?Sized>(value: &T) -> io::Result<Vec<u8>>
where
T: serde::Serialize,
{
bincode::serialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))
}
#[inline]
fn bincode_deserialize<T: ?Sized>(value: &[u8]) -> io::Result<T>
where
T: for<'a> serde::Deserialize<'a>,
{
bincode::deserialize(value).map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))
}
fn get_entry_index_upper_bound<K, V>(keys: &Vec<K>, key: &K) -> usize
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
let mut left = 0;
let mut right = keys.len();
while left < right {
let mid = left + ((right - left) >> 1);
if &keys[mid] <= key {
left = mid + 1;
} else {
right = mid;
}
}
left
}
#[derive(Serialize, Deserialize, Debug, Clone)]
struct BPlusTreeNode<K, V> {
keys: Vec<K>,
children: Vec<Option<BPlusTreeNode<K, V>>>,
children: Vec<BPlusTreeNode<K, V>>,
is_leaf: bool,
values: Vec<Option<V>>, // only used in leaf nodes
values: Vec<V>, // only used in leaf nodes
}
impl<K, V> BPlusTreeNode<K, V>
@@ -27,57 +83,245 @@ where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
#[inline]
fn new(is_leaf: bool) -> Self {
BPlusTreeNode {
is_leaf,
keys: vec![],
children: vec![],
is_leaf,
values: vec![],
}
}
fn insert_non_full(&mut self, key: K, value: V) {
let pos = self.keys.binary_search(&key).unwrap_or_else(|pos| pos);
#[inline]
fn is_overflow(&self, order: usize) -> bool {
self.keys.len() > order
}
if self.is_leaf {
self.keys.insert(pos, key);
self.values.insert(pos, Some(value));
#[inline]
fn get_median_index(order: usize) -> usize {
order >> 1
}
fn find_leaf_entry<'a>(&self, node: &'a BPlusTreeNode<K, V>) -> &'a K {
if node.is_leaf {
node.keys.get(0).unwrap()
} else {
let mut idx = pos;
if self.children[pos].as_mut().unwrap().keys.len() == 2 * T - 1 {
self.split_child(pos);
if key > self.keys[pos] {
idx = pos + 1;
}
}
self.children[idx].as_mut().unwrap().insert_non_full(key, value);
let child = node.children.get(0).unwrap();
self.find_leaf_entry(child)
}
}
fn split_child(&mut self, pos: usize) {
let t = T - 1;
let mut new_node = BPlusTreeNode::new(self.children[pos].as_ref().unwrap().is_leaf);
let mut old_node = self.children[pos].take().unwrap();
#[allow(dead_code)]
fn query(&self, key: &K) -> Option<&V> {
if self.is_leaf {
return match self.keys.binary_search(&key) {
Ok(idx) => self.values.get(idx),
Err(_) => None,
};
}
let node = self.children.get(self.get_entry_index_upper_bound(key)).unwrap();
node.query(key)
}
self.keys.insert(pos, old_node.keys.remove(t));
self.children.insert(pos + 1, Some(new_node.clone()));
fn get_equal_entry_index(&self, key: &K) -> Option<usize>
where
K: Ord,
{
let mut left = 0;
let mut right = self.keys.len().checked_sub(1)?;
while left <= right {
let mid = left + ((right - left) >> 1);
let mid_key = &self.keys[mid];
if mid_key == key {
return Some(mid);
} else if mid_key > key {
right = mid.checked_sub(1)?;
} else {
left = mid + 1;
}
}
None
}
if old_node.is_leaf {
new_node.keys = old_node.keys.split_off(t);
new_node.values = old_node.values.split_off(t);
fn get_entry_index_upper_bound(&self, key: &K) -> usize {
get_entry_index_upper_bound::<K, V>(&self.keys, key)
}
fn insert(&mut self, key: K, v: V, inner_order: usize, leaf_order: usize) -> Option<BPlusTreeNode<K, V>> {
if self.is_leaf {
if let Some(eq_entry_index) = self.get_equal_entry_index(&key) {
self.values.insert(eq_entry_index, v);
return None;
}
let pos = self.get_entry_index_upper_bound(&key);
self.keys.insert(pos, key);
self.values.insert(pos, v);
if self.is_overflow(leaf_order) {
return Some(self.split(leaf_order));
}
} else {
new_node.keys = old_node.keys.split_off(t + 1);
new_node.children = old_node.children.split_off(t + 1);
let pos = self.get_entry_index_upper_bound(&key);
let child = self.children.get_mut(pos).unwrap();
let node = child.insert(key, v, inner_order, leaf_order);
if node.is_some() {
let leaf_key = self.find_leaf_entry(node.as_ref().unwrap());
let idx = self.get_entry_index_upper_bound(leaf_key);
self.keys.insert(idx, leaf_key.clone());
self.children.insert(idx + 1, node.unwrap());
if self.is_overflow(inner_order) {
return Some(self.split(inner_order));
}
}
}
None
}
fn split(&mut self, order: usize) -> BPlusTreeNode<K, V> {
let median = BPlusTreeNode::<K, V>::get_median_index(order);
if self.is_leaf {
let mut node = BPlusTreeNode::new(true);
node.keys = self.keys.split_off(median);
node.values = self.values.split_off(median);
node
} else {
let mut node = BPlusTreeNode::new(false);
node.keys = self.keys.split_off(median + 1);
node.children = self.children.split_off(median + 1);
self.children.push(node.children.get(0).unwrap().clone());
node
}
}
pub(crate) fn traverse<F>(&self, visit: &mut F)
where
F: FnMut(&Vec<K>, &Vec<V>),
{
if self.is_leaf {
visit(&self.keys, &self.values);
}
self.children.iter().for_each(|child| child.traverse(visit));
}
fn serialize_to_block<W: Write + Seek>(&self, file: &mut W, buffer: &mut Vec<u8>, offset: u64) -> io::Result<u64> {
let mut current_offset = offset;
let buffer_slice = &mut buffer[..];
// Write node type (leaf or internal)
buffer_slice[0] = if self.is_leaf { 1u8 } else { 0u8 };
let mut write_pos = 1;
// Serialize and write keys
let keys_encoded = bincode_serialize(&self.keys)?;
let keys_bytes_len = keys_encoded.len();
buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(keys_bytes_len as u32).to_le_bytes());
write_pos += 4;
buffer_slice[write_pos..write_pos + keys_bytes_len].copy_from_slice(&keys_encoded);
write_pos += keys_bytes_len;
// If leaf, serialize and write values
if self.is_leaf {
let values_encoded = bincode_serialize(&self.values)?;
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::fast());
encoder.write_all(&values_encoded)?;
let compressed_bytes = encoder.finish()?;
let values_bytes_len = compressed_bytes.len();
buffer_slice[write_pos..write_pos + 4].copy_from_slice(&(values_bytes_len as u32).to_le_bytes());
write_pos += 4;
buffer_slice[write_pos..write_pos + values_bytes_len].copy_from_slice(&compressed_bytes);
write_pos += values_bytes_len;
}
self.children[pos] = Some(old_node);
self.children[pos + 1] = Some(new_node);
// Write the complete buffer to file, do not optimize to real filled size,
file.seek(SeekFrom::Start(offset))?;
file.write_all(&buffer_slice[..BLOCK_SIZE])?; // use BLOCK_SIZE
current_offset += BLOCK_SIZE as u64;
if !self.is_leaf {
let pointer_offset = offset + write_pos as u64;
let mut pointer = Vec::with_capacity(self.children.len());
for child in &self.children {
pointer.push(current_offset);
current_offset = child.serialize_to_block(file, buffer, current_offset)?;
}
let pointer_encoded = bincode_serialize(&pointer)?;
let pointer_bytes_len = pointer_encoded.len() as u32;
file.seek(SeekFrom::Start(pointer_offset))?;
file.write_all(&pointer_bytes_len.to_le_bytes())?;
file.write_all(&pointer_encoded)?;
}
Ok(current_offset)
}
fn deserialize_from_block<R: Read + Seek>(file: &mut R, buffer: &mut Vec<u8>, offset: u64, nested: bool) -> io::Result<(Self, Option<Vec<u64>>)> {
file.seek(SeekFrom::Start(offset))?;
file.read_exact(buffer)?;
// Read the node type directly from buffer
let is_leaf = buffer[0] == 1u8;
let mut read_pos = 1;
// Deserialize keys
let keys_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize;
read_pos += 4;
let keys: Vec<K> = bincode_deserialize(&buffer[read_pos..read_pos + keys_length])?;
read_pos += keys_length;
// Deserialize values if leaf node
let values = if is_leaf {
let values_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize;
read_pos += 4;
let compressed_bytes = &buffer[read_pos..read_pos + values_length];
let mut decoder = flate2::write::ZlibDecoder::new(Vec::new());
decoder.write_all(&compressed_bytes)?;
let values_bytes = decoder.finish()?;
let values: Vec<V> = bincode_deserialize(&values_bytes)?;
read_pos += values_length;
values
} else {
vec![]
};
// Deserialize children indices if internal node
let (children, children_pointer) = if !is_leaf {
let pointers_length = u32_from_bytes(&buffer[read_pos..read_pos + 4])? as usize;
read_pos += 4;
let pointers: Vec<u64> = bincode_deserialize(&buffer[read_pos..read_pos + pointers_length])?;
if nested {
let nodes: Result<Vec<BPlusTreeNode<K, V>>, io::Error> = pointers
.iter()
.map(|pointer| {
BPlusTreeNode::<K, V>::deserialize_from_block(file, buffer, *pointer, nested)
.map(|(node, _)| node)
.map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))
})
.collect();
(nodes?, None)
} else {
(vec![], Some(pointers))
}
} else {
(vec![], None)
};
Ok((BPlusTreeNode {
is_leaf,
keys,
values,
children,
}, children_pointer))
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
struct BPlusTree<K, V> {
pub(crate) struct BPlusTree<K, V> {
root: BPlusTreeNode<K, V>,
inner_order: usize,
leaf_order: usize,
}
impl<K, V> BPlusTree<K, V>
@@ -86,96 +330,287 @@ where
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
pub(crate) fn new() -> Self {
let key_size = size_of::<K>() + POINTER_SIZE + size_of::<bool>() + BINCODE_OVERHEAD;
let inner_order = BLOCK_SIZE / key_size;
let leaf_order = BLOCK_SIZE / (key_size + size_of::<V>() + BINCODE_OVERHEAD);
BPlusTree {
root: BPlusTreeNode::new(true),
root: BPlusTreeNode::<K, V>::new(true),
inner_order,
leaf_order,
}
}
fn new_with_root(root: BPlusTreeNode::<K, V>) -> Self {
let key_size = size_of::<K>() + POINTER_SIZE + size_of::<bool>() + BINCODE_OVERHEAD;
let inner_order = BLOCK_SIZE / key_size;
let leaf_order = BLOCK_SIZE / (key_size + size_of::<V>() + BINCODE_OVERHEAD);
BPlusTree {
root,
inner_order,
leaf_order,
}
}
pub(crate) fn insert(&mut self, key: K, value: V) {
if self.root.keys.len() == 2 * T - 1 {
let mut new_root = BPlusTreeNode::new(false);
new_root.children.push(Some(self.root.clone()));
new_root.split_child(0);
if self.root.keys.is_empty() {
self.root.keys.push(key);
self.root.values.push(value);
return;
}
if let Some(node) = self.root.insert(key, value, self.inner_order, self.leaf_order) {
let child_key = if node.is_leaf {
node.keys.get(0).as_ref().unwrap()
} else {
node.find_leaf_entry(&node)
};
let mut new_root = BPlusTreeNode::<K, V>::new(false);
new_root.keys.push(child_key.clone());
new_root.children.push(std::mem::replace(&mut self.root, BPlusTreeNode::new(true))); // `true` als Beispiel für ein Blatt
new_root.children.push(node);
self.root = new_root;
}
self.root.insert_non_full(key, value);
}
pub(crate) fn query(&self, key: &K) -> Option<V> {
let mut node = &self.root;
while !node.is_leaf {
let pos = match node.keys.binary_search(key) {
Ok(pos) => return node.values[pos].clone(),
Err(pos) => pos,
#[allow(dead_code)]
pub(crate) fn query(&self, key: &K) -> Option<&V> {
self.root.query(key)
}
pub(crate) fn serialize(&self, filepath: &Path) -> io::Result<u64> {
let mut file = OpenOptions::new().write(true).create(true).open(filepath)?;
let mut buffer = vec![0u8; BLOCK_SIZE];
let result = self.root.serialize_to_block(&mut file, &mut buffer, 0u64);
file.flush()?;
result
}
pub(crate) fn deserialize(filepath: &Path) -> io::Result<Self> {
let mut file = File::open(filepath)?;
if is_multiple_of_block_size(&file).is_err() {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Tree file has to be multiple of block size {BLOCK_SIZE}")));
}
let mut buffer = vec![0u8; BLOCK_SIZE];
let (root, _) = BPlusTreeNode::deserialize_from_block(&mut file, &mut buffer, 0, true)?;
Ok(BPlusTree::new_with_root(root))
}
pub(crate) fn traverse<F>(&self, mut visit: F)
where
F: FnMut(&Vec<K>, &Vec<V>),
{
self.root.traverse(&mut visit);
}
}
fn query_tree<K, V>(file: &mut File, key: &K) -> Option<V>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
let mut offset = 0;
let mut buffer = vec![0u8; BLOCK_SIZE];
loop {
match BPlusTreeNode::<K, V>::deserialize_from_block(file, &mut buffer, offset, false) {
Ok((node, pointers)) => {
if node.is_leaf {
return match node.keys.binary_search(key) {
Ok(idx) => node.values.get(idx).cloned(),
Err(_) => None,
};
}
let child_idx = get_entry_index_upper_bound::<K, V>(&node.keys, key);
offset = *pointers.unwrap().get(child_idx).unwrap();
}
Err(err) => {
error!("Failed to read id tree from file {err}");
return None;
}
};
}
}
pub(crate) struct BPlusTreeQuery<K, V> {
file: File,
_marker_k: PhantomData<K>,
_marker_v: PhantomData<V>,
}
impl<K, V> BPlusTreeQuery<K, V>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
pub(crate) fn try_new(filepath: &Path) -> io::Result<Self> {
let file = is_file_valid(File::open(filepath)?)?;
Ok(BPlusTreeQuery {
file,
_marker_k: Default::default(),
_marker_v: Default::default(),
})
}
pub(crate) fn query(&mut self, key: &K) -> Option<V> {
query_tree(&mut self.file, key)
}
}
pub(crate) struct BPlusTreeUpdate<K, V> {
file: File,
_marker_k: PhantomData<K>,
_marker_v: PhantomData<V>,
}
impl<K, V> BPlusTreeUpdate<K, V>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
pub(crate) fn try_new(filepath: &Path) -> io::Result<Self> {
if !filepath.exists() {
return Err(io::Error::new(io::ErrorKind::NotFound, format!("File not found {}", filepath.to_str().unwrap_or("?"))));
}
let file = OpenOptions::new()
.write(true)
.read(true)
.open(filepath)?;
let file = is_file_valid(file)?;
Ok(BPlusTreeUpdate {
file,
_marker_k: Default::default(),
_marker_v: Default::default(),
})
}
pub(crate) fn query(&mut self, key: &K) -> Option<V> {
query_tree(&mut self.file, key)
}
fn serialize_node(&mut self, offset: u64, node: BPlusTreeNode<K, V>) -> io::Result<u64> {
let mut buffer = vec![0u8; BLOCK_SIZE];
let result = node.serialize_to_block(&mut self.file, &mut buffer, offset);
self.file.flush()?;
result
}
pub(crate) fn update(&mut self, key: &K, value: V) -> io::Result<u64> {
let mut offset = 0;
let mut buffer = vec![0u8; BLOCK_SIZE];
loop {
match BPlusTreeNode::<K, V>::deserialize_from_block(&mut self.file, &mut buffer, offset, false) {
Ok((mut node, pointers)) => {
if node.is_leaf {
return match node.keys.binary_search(key) {
Ok(idx) => {
let old_value = node.values.get(idx);
if let Some(_) = old_value {
node.values[idx] = value;
return self.serialize_node(offset, node);
}
Err(io::Error::new(io::ErrorKind::NotFound, "Entry not found"))
}
Err(_) => Err(io::Error::new(io::ErrorKind::NotFound, "Entry not found")),
};
}
let child_idx = get_entry_index_upper_bound::<K, V>(&node.keys, key);
offset = *pointers.unwrap().get(child_idx).unwrap();
}
Err(err) => {
error!("Failed to read id tree from file {err}");
return Err(io::Error::new(io::ErrorKind::NotFound, format!("Failed to read id tree from file {err}")));
}
};
node = node.children[pos].as_ref().unwrap();
}
match node.keys.binary_search(key) {
Ok(pos) => node.values[pos].clone(),
Err(_) => None,
}
}
pub(crate) fn serialize_to_file(&self, filename: &str) -> io::Result<()> {
let file = File::create(filename)?;
let encoder = GzEncoder::new(file, Compression::default());
match bincode::serialize_into(encoder, &self) {
Ok(()) => Ok(()),
Err(e) => {
println!("Failed to write bplustree to disk {e}");
Ok(())
}
}
}
// If file exists the file is deserialized, otherwise an empty tree is returned
pub(crate) fn deserialize_from_file(filename: &str) -> Self {
match File::open(filename) {
Ok(file) => {
let decoder = GzDecoder::new(file);
let tree: BPlusTree<K, V> = bincode::deserialize_from(decoder).unwrap();
tree
}
Err(_) => BPlusTree::new()
}
}
}
#[cfg(test)]
mod tests {
use std::io;
use serde::{Deserialize, Serialize};
use crate::repository::bplustree::{BPlusTree};
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate};
// Example usage with a simple struct
#[derive(Serialize, Deserialize, Debug, Clone)]
struct Value {
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
struct Record {
id: u32,
data: String,
}
#[test]
fn insert_test() -> io::Result<()> {
let mut tree = BPlusTree::new();
tree.insert("abc".to_string(), Value { id: 16, data: "one".to_string() });
tree.insert("def".to_string(), Value { id: 32, data: "two".to_string() });
tree.insert("ghi".to_string(), Value { id: 64, data: "three".to_string() });
let mut tree = BPlusTree::<u32, Record>::new();
for i in 0u32..=500 {
tree.insert(i, Record {
id: i,
data: format!("Entry {i}"),
});
}
// // Traverse the tree
// tree.traverse(|node| {
// println!("Node: {:?}", node);
// });
let filepath = PathBuf::from("/tmp/tree.bin");
// Serialize the tree to a file
tree.serialize_to_file("/tmp/tree.bin")?;
tree.serialize(&filepath)?;
// Deserialize the tree from the file
let tree = BPlusTree::<String, Value>::deserialize_from_file("/tmp/tree.bin");
tree = BPlusTree::<u32, Record>::deserialize(&filepath)?;
// Query the tree
if let Some(value) = tree.query(&("ghi".to_string())) {
println!("Found: {:?}", value);
} else {
println!("Not found");
for i in 0u32..=500 {
let found = tree.query(&i);
assert!(found.is_some(), "Entry {} not found", i);
assert!(found.unwrap().eq(&Record {
id: i,
data: format!("Entry {i}"),
}), "Entry {} not found", i);
}
let mut tree_query: BPlusTreeQuery<u32, Record> = BPlusTreeQuery::try_new(&filepath)?;
for i in 0u32..=500 {
let found = tree_query.query(&i);
assert!(found.is_some(), "Entry {} not found", i);
let entry = found.unwrap();
assert!(entry.eq(&Record {
id: i,
data: format!("Entry {i}"),
}), "Entry {} not found", i);
}
let mut tree_update: BPlusTreeUpdate<u32, Record> = BPlusTreeUpdate::try_new(&filepath)?;
for i in 0u32..=500 {
if let Some(record) = tree_update.query(&i) {
let new_record = Record {
id: record.id,
data: format!("Entry {}", record.id + 9000),
};
tree_update.update(&i, new_record)?;
} else {
assert!(false, "Entry {} not found", i);
}
}
let mut tree_query: BPlusTreeQuery<u32, Record> = BPlusTreeQuery::try_new(&filepath)?;
for i in 0u32..=500 {
let found = tree_query.query(&i);
assert!(found.is_some(), "Entry {} not found", i);
let entry = found.unwrap();
let expected = Record {
id: i,
data: format!("Entry {}", i + 9000),
};
assert!(entry.eq(&expected), "Entry not equal {:?} != {:?}", entry, expected);
}
Ok(())
}
}
}
src/repository/epg_repository.rs
+2 -2
View File
@@ -42,7 +42,7 @@ fn epg_write_file(target: &ConfigTarget, epg: &Epg, path: &Path) -> Result<(), M
Ok(())
}
pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, epg: Option<&Epg>, output: &TargetOutput) -> Result<(), M3uFilterError> {
pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, target_path: &Path, epg: Option<&Epg>, output: &TargetOutput) -> Result<(), M3uFilterError> {
if let Some(epg_data) = epg {
match &output.target {
TargetType::M3u => {
@@ -51,7 +51,7 @@ pub(crate) fn epg_write(target: &ConfigTarget, cfg: &Config, epg: Option<&Epg>,
M3uFilterErrorKind::Notify,
format!("write epg for target {} failed: No filename set", target.name)));
}
if let Some(path) = m3u_get_epg_file_path(cfg, target) {
if let Some(path) = m3u_get_epg_file_path(target_path) {
if log_enabled!(Level::Debug) {
debug!("writing m3u epg to {}", path.to_str().unwrap_or("?"));
}
src/repository/id_mapping.rs
+84
View File
@@ -0,0 +1,84 @@
// This module is not included in the build
use std::cmp::max;
use std::io::Error;
use std::path::{Path, PathBuf};
use log::error;
use serde::{Deserialize, Serialize};
use crate::repository::bplustree::BPlusTree;
pub(crate) struct IdMapping<V>
where
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
dirty: bool,
tree: BPlusTree<u32, V>,
path: PathBuf,
max_id: u32,
}
impl<V> IdMapping<V>
where
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
pub(crate) fn new(path: &Path) -> Self {
let tree: BPlusTree<u32, V> = match BPlusTree::<u32, V>::deserialize(&path) {
Ok(tree) => tree,
_ => BPlusTree::<u32, V>::new()
};
let mut max_id = 0;
tree.traverse(|keys, _| {
match keys.iter().max() {
None => {}
Some(max_value) => {
max_id = max(max_id, *max_value);
}
}
});
IdMapping {
dirty: false,
tree,
path: path.to_path_buf(),
max_id,
}
}
pub(crate) fn insert(&mut self, id: u32, value: V) {
self.dirty = true;
self.tree.insert(id, value);
}
pub(crate) fn persist(&mut self) -> Result<(), Error> {
if self.dirty {
self.tree.serialize(&self.path)?;
}
self.dirty = false;
Ok(())
}
pub(crate) fn max_id(&self) -> u32 {
self.max_id
}
pub(crate) fn query(&self, id: u32) -> Option<&V> {
self.tree.query(&id)
}
}
impl<V> Drop for IdMapping<V>
where
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
fn drop(&mut self) {
match self.persist() {
Ok(_) => {}
Err(err) => {
error!("Failed to persist id mapping {:?} err:{}", &self.path, err.to_string())
}
}
}
}
src/repository/index_record.rs
-70
View File
@@ -1,70 +0,0 @@
use std::fs::File;
use std::io::{Error, Read, Seek, SeekFrom, Write};
use std::path::Path;
use crate::utils::file_utils;
/**
We write the structs with `bincode::encode` to a file.
To access each entry we need a index file where we can find the
Entries with offset and size of the encoded struct.
This is used for the index file where first entry is the index
of the encoded file, and size is the size of the encoded struct.
We also use it for different purposes, like storing id -> to id mapping.
*/
pub(in crate::repository) struct IndexRecord {
pub left: u32,
pub right: u32,
}
impl IndexRecord {
pub fn from_file(file: &mut File, offset: u32) -> Result<Self, Error> {
file.seek(SeekFrom::Start(u64::from(offset)))?;
let mut left_bytes = [0u8; 4];
let mut right_bytes = [0u8; 4];
file.read_exact(&mut left_bytes)?;
file.read_exact(&mut right_bytes)?;
let left = u32::from_le_bytes(left_bytes);
let right = u32::from_le_bytes(right_bytes);
Ok(IndexRecord { left, right })
}
pub fn to_file(path: &Path, left: u32, right: u32, append: bool) -> Result<(), Error> {
match file_utils::open_file_append(path, append) {
Ok(mut file) => {
let bytes = IndexRecord::to_bytes(left, right);
file.write_all(&bytes)
},
Err(err) => Err(err)
}
}
// pub fn from_bytes(bytes: &[u8], cursor: &mut usize) -> Result<Self, Error> {
// if let Ok(index_bytes) = bytes[*cursor..*cursor + 4].try_into() {
// *cursor += 4;
// if let Ok(size_bytes) = bytes[*cursor..*cursor + 2].try_into() {
// *cursor += 2;
// let index = u32::from_le_bytes(index_bytes);
// let size = u16::from_le_bytes(size_bytes);
// return Ok(IndexRecord { index, size });
// }
// }
// Err(Error::new(ErrorKind::Other, "Failed to read index"))
// }
// pub fn as_bytes(&self) -> [u8; 6] {
// IndexRecord::to_bytes(self.index, self.size)
// }
pub fn to_bytes(left: u32, right: u32) -> [u8; 8] {
let left_bytes: [u8; 4] = left.to_le_bytes();
let right_bytes: [u8; 4] = right.to_le_bytes();
let mut combined_bytes: [u8; 8] = [0; 8];
combined_bytes[..4].copy_from_slice(&left_bytes);
combined_bytes[4..].copy_from_slice(&right_bytes);
combined_bytes
}
pub fn get_record_size() -> u32 { 8 }
pub fn get_index_offset(index: u32) -> u32 { index * 8 }
}
src/repository/indexed_document_reader.rs
+77 -68
View File
@@ -3,12 +3,30 @@ use std::fs::File;
use std::io::{Error, ErrorKind, Read, Seek, SeekFrom};
use std::marker::PhantomData;
use std::path::Path;
use crate::repository::bplustree::BPlusTreeQuery;
use crate::repository::indexed_document_writer::OffsetPointer;
use crate::repository::index_record::IndexRecord;
fn get_offset(index_path: &Path, doc_id: u32) -> Result<u64, Error> {
match BPlusTreeQuery::<u32,OffsetPointer>::try_new(index_path) {
Ok(mut tree) => {
match tree.query(&doc_id) {
Some(offset) => Ok(offset as u64),
None => Err(Error::new(ErrorKind::NotFound, format!("doc_id not found {doc_id}"))),
}
},
Err(err) => Err(err)
}
}
fn read_content_size(main_file: &mut File) -> Result<usize, Error> {
let mut size_bytes = [0u8; 4];
main_file.read_exact(&mut size_bytes)?;
let buf_size = u32::from_le_bytes(size_bytes) as usize;
Ok(buf_size)
}
pub(in crate::repository) struct IndexedDocumentReader<T> {
main_file: File,
index_file: File,
cursor: u32,
size: u32,
failed: bool,
@@ -17,37 +35,31 @@ pub(in crate::repository) struct IndexedDocumentReader<T> {
}
impl<T: ?Sized + serde::de::DeserializeOwned> IndexedDocumentReader<T> {
pub fn new(main_path: &Path, index_path: &Path) -> Result<IndexedDocumentReader<T>, Error> {
if main_path.exists() && index_path.exists() {
pub fn new(main_path: &Path) -> Result<IndexedDocumentReader<T>, Error> {
if main_path.exists() {
match File::open(main_path) {
Ok(main_file) => {
match File::open(index_path) {
Ok(index_file) => {
let size = match index_file.metadata() {
Ok(metadata) => {
usize::try_from(metadata.len()).map_err(|err| Error::new(ErrorKind::Other, err))?
}
Err(_e) => 0,
};
Ok(Self {
main_file,
index_file,
cursor: 0,
size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?,
failed: false,
t_buffer: Vec::new(),
t_type: PhantomData,
})
let size = match main_file.metadata() {
Ok(metadata) => {
usize::try_from(metadata.len()).map_err(|err| Error::new(ErrorKind::Other, err))?
}
Err(e) => Err(e)
}
Err(_e) => 0,
};
Ok(Self {
main_file,
cursor: 0,
size: u32::try_from(size).map_err(|err| Error::new(ErrorKind::Other, err))?,
failed: false,
t_buffer: Vec::new(),
t_type: PhantomData,
})
}
Err(e) => Err(e)
}
} else {
Err(Error::new(ErrorKind::NotFound, format!("File not found {} or {}",
main_path.to_str().unwrap(),
index_path.to_str().unwrap())))
Err(Error::new(ErrorKind::NotFound, format!("File not found {}",
main_path.to_str().unwrap())))
}
}
@@ -59,33 +71,47 @@ impl<T: ?Sized + serde::de::DeserializeOwned> IndexedDocumentReader<T> {
!self.failed && self.cursor < self.size
}
pub fn read_next(&mut self) -> Result<Option<T>, Error> {
if self.has_next() {
let record = IndexRecord::from_file(&mut self.index_file, self.cursor);
self.cursor += IndexRecord::get_record_size();
match record {
Ok(index_record) => {
let offset = u64::from(index_record.left);
let buf_size = index_record.right as usize;
if self.t_buffer.len() < buf_size {
self.t_buffer.resize(buf_size, 0u8);
}
self.main_file.seek(SeekFrom::Start(offset))?;
self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?;
return match bincode::deserialize::<T>(&self.t_buffer[0..buf_size]) {
Ok(value) => Ok(Some(value)),
Err(err) => {
self.failed = true;
Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}")))
}
};
}
Err(err) => {
self.failed = true;
return Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}")));
}
if !self.has_next() {
return Ok(None);
}
// read content-size
let buf_size: usize = read_content_size(&mut self.main_file)?;
self.cursor += 4;
// resize buffer if necessary
if self.t_buffer.capacity() < buf_size {
self.t_buffer.reserve(buf_size - self.t_buffer.capacity());
}
self.t_buffer.resize(buf_size, 0u8);
// read content
self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?;
self.cursor += buf_size as u32;
// deserialize buffer
return match bincode::deserialize::<T>(&self.t_buffer[0..buf_size]) {
Ok(value) => Ok(Some(value)),
Err(err) => {
self.failed = true;
Err(Error::new(ErrorKind::Other, format!("Failed to deserialize document {err}")))
}
};
}
pub(in crate::repository) fn read_indexed_item(main_path: &Path, index_path: &Path, doc_id: u32) -> Result<T, Error>
{
if main_path.exists() && index_path.exists() {
// get the offset from index
let offset = crate::repository::indexed_document_reader::get_offset(index_path, doc_id)?;
let mut main_file = File::open(main_path)?;
main_file.seek(SeekFrom::Start(offset))?;
let buf_size = read_content_size(&mut main_file)?;
let mut buffer: Vec<u8> = vec![0; buf_size];
main_file.read_exact(&mut buffer)?;
if let Ok(item) = bincode::deserialize::<T>(&buffer) {
return Ok(item);
}
}
Ok(None)
Err(Error::new(ErrorKind::Other, format!("Failed to read item for id {} - {}", doc_id, main_path.to_str().unwrap())))
}
}
@@ -102,20 +128,3 @@ impl<T: ?Sized + serde::de::DeserializeOwned> Iterator for IndexedDocumentReader
None
}
}
pub(in crate::repository) fn read_indexed_item<T>(main_path: &Path, index_path: &Path, offset: u32) -> Result<T, Error>
where T: ?Sized + serde::de::DeserializeOwned
{
if main_path.exists() && index_path.exists() {
let mut index_file = File::open(index_path)?;
let mut main_file = File::open(main_path)?;
let index_record = IndexRecord::from_file(&mut index_file, offset)?;
main_file.seek(SeekFrom::Start(u64::from(index_record.left)))?;
let mut buffer: Vec<u8> = vec![0; index_record.right as usize];
main_file.read_exact(&mut buffer)?;
if let Ok(item) = bincode::deserialize::<T>(&buffer[..]) {
return Ok(item);
}
}
Err(Error::new(ErrorKind::Other, format!("Failed to read item for offset {} - {}", offset, main_path.to_str().unwrap())))
}
src/repository/indexed_document_writer.rs
+43 -24
View File
@@ -2,39 +2,56 @@ use std::convert::TryFrom;
use std::fs::File;
use std::io::{Error, ErrorKind, Write};
use std::path::PathBuf;
use log::error;
use crate::repository::bplustree::BPlusTree;
use crate::repository::index_record::IndexRecord;
use crate::utils::file_utils;
use crate::utils::file_utils::create_file_tuple;
use crate::utils::file_utils::{open_file_append};
pub(in crate::repository) type OffsetPointer = u32;
/**
* Creates two files,
* - content
* - index
*
* Layout of content file record is:
* - content-size (u32) + content
*
* index file is a bplustree
*/
pub(in crate::repository) struct IndexedDocumentWriter {
main_path: PathBuf,
index_path: PathBuf,
main_file: File,
index_file: File,
main_offset: u32,
index_offset: u32,
main_offset: OffsetPointer,
index_tree: BPlusTree<u32, OffsetPointer>,
}
impl IndexedDocumentWriter {
fn new_with_mode(main_path: PathBuf, index_path: PathBuf, append: bool) -> Result<Self, Error> {
match create_file_tuple(&main_path, &index_path, append) {
Ok((main_file, index_file)) => {
match open_file_append(&main_path, append) {
Ok(main_file) => {
let main_offset = match &main_file.metadata() {
Ok(meta) => u32::try_from(meta.len()).map_err(|err| Error::new(ErrorKind::Other, err))?,
Err(_) => 0
};
let index_offset = match &index_file.metadata() {
Ok(meta) => u32::try_from(meta.len()).map_err(|err| Error::new(ErrorKind::Other, err))?,
Err(_) => 0
let index_tree = if append && index_path.exists() {
BPlusTree::<u32, OffsetPointer>::deserialize(&index_path).unwrap_or_else(|err| {
error!("Failed to load index {:?} {err}", index_path);
BPlusTree::<u32, OffsetPointer>::new()
})
} else {
BPlusTree::<u32, OffsetPointer>::new()
};
Ok(Self {
main_path,
index_path,
main_file,
index_file,
main_offset,
index_offset,
index_tree
})
}
Err(e) => Err(e)
@@ -49,27 +66,29 @@ impl IndexedDocumentWriter {
Self::new_with_mode(main_path, index_path, true)
}
pub fn write_doc<T>(&mut self, doc: &T) -> Result<(u32, u32), Error>
pub fn flush(&mut self) -> std::io::Result<()> {
self.main_file.flush()?;
self.index_tree.serialize(&self.index_path).map(|_| ())
}
pub fn write_doc<T>(&mut self, doc_id: u32, doc: &T) -> Result<(), Error>
where
T: ?Sized + serde::Serialize {
let current_main_index = self.main_offset;
let current_index_index = self.index_offset;
if let Ok(encoded) = bincode::serialize(doc) {
match file_utils::check_write(&self.main_file.write_all(&encoded)) {
let content_bytes_len = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?;
let mut data: Vec<u8> = content_bytes_len.to_le_bytes().to_vec();
data.extend(&encoded);
match file_utils::check_write(&self.main_file.write_all(&data)) {
Ok(()) => {
let bytes_written = u32::try_from(encoded.len()).map_err(|err| Error::new(ErrorKind::Other, err))?;
let combined_bytes = IndexRecord::to_bytes(self.main_offset, bytes_written);
if let Err(err) = file_utils::check_write(&self.index_file.write_all(&combined_bytes)) {
return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.index_path.to_str().unwrap(), err)));
}
self.main_offset += bytes_written;
self.index_offset += IndexRecord::get_record_size();
self.index_tree.insert(doc_id, self.main_offset);
let written_bytes = u32::try_from(data.len()).map_err(|err| Error::new(ErrorKind::Other, err))?;
self.main_offset += written_bytes;
}
Err(err) => {
return Err(Error::new(ErrorKind::Other, format!("failed to write document: {} - {}", self.main_path.to_str().unwrap(), err)));
}
}
}
Ok((current_main_index, current_index_index))
Ok(())
}
}
src/repository/m3u_repository.rs
+85 -65
View File
@@ -1,33 +1,32 @@
use std::fs::File;
use std::io::{BufWriter, Error, ErrorKind, Write};
use std::path::{Path, PathBuf};
use std::rc::Rc;
use log::error;
use crate::{create_m3u_filter_error_result};
use crate::api::api_utils::get_user_server_info;
use crate::create_m3u_filter_error;
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::model::api_proxy::{ProxyType, ProxyUserCredentials};
use crate::model::config::{Config, ConfigTarget};
use crate::model::playlist::{M3uPlaylistItem, PlaylistGroup, PlaylistItem, PlaylistItemType};
use crate::repository::index_record::IndexRecord;
use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item};
use crate::repository::indexed_document_reader::IndexedDocumentReader;
use crate::repository::indexed_document_writer::IndexedDocumentWriter;
use crate::repository::storage::ensure_target_storage_path;
use crate::utils::file_utils;
macro_rules! cant_write_result {
($path:expr, $err:expr) => {
create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err)
create_m3u_filter_error!(M3uFilterErrorKind::Notify, "failed to write m3u playlist: {} - {}", $path.to_str().unwrap() ,$err)
}
}
fn m3u_get_base_file_path(cfg: &Config, target: &ConfigTarget) -> Option<PathBuf> {
file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(format!("m3u_{}.db", target.name.replace(' ', "_").as_str()))))
fn m3u_get_base_file_path(target_path: &Path) -> Option<PathBuf> {
Some(target_path.join(PathBuf::from("m3u.db")))
}
pub(crate) fn m3u_get_file_paths(cfg: &Config, target: &ConfigTarget) -> Option<(PathBuf, PathBuf)> {
match m3u_get_base_file_path(cfg, target) {
pub(crate) fn m3u_get_file_paths(target_path: &Path) -> Option<(PathBuf, PathBuf)> {
match m3u_get_base_file_path(target_path) {
Some(m3u_path) => {
let extension = m3u_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or("")));
let index_path = m3u_path.with_extension(format!("{}idx", &extension.unwrap_or(String::new())));
@@ -37,93 +36,114 @@ pub(crate) fn m3u_get_file_paths(cfg: &Config, target: &ConfigTarget) -> Option<
}
}
pub(crate) fn m3u_get_epg_file_path(cfg: &Config, target: &ConfigTarget) -> Option<PathBuf> {
m3u_get_base_file_path(cfg, target)
pub(crate) fn m3u_get_epg_file_path(target_path: &Path) -> Option<PathBuf> {
m3u_get_base_file_path(target_path)
.map(|path| file_utils::add_prefix_to_filename(&path, "epg_", Some("xml")))
}
pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> {
if !new_playlist.is_empty() {
if let Some((m3u_path, idx_path)) = m3u_get_file_paths(cfg, target) {
fn persist_m3u_playlist_as_text(target: &ConfigTarget, cfg: &Config, m3u_playlist: &Vec<M3uPlaylistItem>) {
if let Some(filename) = target.get_m3u_filename() {
if let Some(m3u_filename) = file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) {
match File::create(&m3u_filename) {
Ok(file) => {
let mut buf_writer = BufWriter::new(file);
let _ = buf_writer.write("#EXTM3U\n".as_bytes());
for m3u in m3u_playlist {
let _ = buf_writer.write(m3u.to_m3u(target, None).as_bytes());
let _ = buf_writer.write("\n".as_bytes());
}
}
Err(_) => {
error!("Can't write m3u plain playlist {}", &m3u_filename.to_str().unwrap());
}
}
}
}
}
pub(crate) fn m3u_write_playlist(target: &ConfigTarget, cfg: &Config, target_path: &Path, new_playlist: &[PlaylistGroup]) -> Result<(), M3uFilterError> {
if !new_playlist.is_empty() {
if let Some((m3u_path, idx_path)) = m3u_get_file_paths(target_path) {
let m3u_playlist = new_playlist.iter()
.flat_map(|pg| &pg.channels)
.filter(|&pli| pli.header.borrow().item_type != PlaylistItemType::SeriesInfo)
.map(PlaylistItem::to_m3u).collect::<Vec<M3uPlaylistItem>>();
if let Some(filename) = target.get_m3u_filename() {
if let Some(m3u_filename) = file_utils::get_file_path(&cfg.working_dir, Some(PathBuf::from(filename))) {
match File::create(&m3u_filename) {
Ok(file) => {
let mut buf_writer = BufWriter::new(file);
let _ = buf_writer.write("#EXTM3U\n".as_bytes());
for m3u in &m3u_playlist {
let _ = buf_writer.write(m3u.to_m3u(target, None).as_bytes());
let _ = buf_writer.write("\n".as_bytes());
persist_m3u_playlist_as_text(target, cfg, &m3u_playlist);
{
let _file_lock = cfg.file_locks.write_lock(&m3u_path).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, format!("{}", err)))?;
match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) {
Ok(mut writer) => {
for m3u in m3u_playlist {
match writer.write_doc(m3u.virtual_id, &m3u) {
Ok(_) => {}
Err(err) => return Err(cant_write_result!(&m3u_path, err))
}
}
Err(_) => {
error!("Can't write m3u plain playlist {}", &m3u_filename.to_str().unwrap());
}
writer.flush().map_err(|err| cant_write_result!(&m3u_path, err))?;
}
Err(err) => return Err(cant_write_result!(&m3u_path, err))
}
}
match IndexedDocumentWriter::new(m3u_path.clone(), idx_path) {
Ok(mut writer) => {
let mut stream_id: u32 = 1;
for mut m3u in m3u_playlist {
m3u.stream_id = Rc::new(stream_id.to_string());
match writer.write_doc(&m3u) {
Ok(_) => stream_id += 1,
Err(err) => return cant_write_result!(&m3u_path, err)
}
}
}
Err(err) => return cant_write_result!(&m3u_path, err)
}
}
}
Ok(())
}
pub(crate) fn m3u_load_rewrite_playlist(cfg: &Config, target: &ConfigTarget, user: &ProxyUserCredentials) -> Option<String> {
if let Some((m3u_path, idx_path)) = m3u_get_file_paths(cfg, target) {
match IndexedDocumentReader::<M3uPlaylistItem>::new(&m3u_path, &idx_path) {
Ok(mut reader) => {
let server_info = get_user_server_info(cfg, user);
let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password);
let mut result = vec![];
result.push("#EXTM3U".to_string());
for m3u_pli in reader.by_ref() {
match user.proxy {
ProxyType::Reverse => {
let stream_id = Rc::clone(&m3u_pli.stream_id);
result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{stream_id}").as_str())));
match ensure_target_storage_path(cfg, target.name.as_str()) {
Ok(target_path) => {
if let Some((m3u_path, _)) = m3u_get_file_paths(&target_path) {
{
let _file_lock = cfg.file_locks.read_lock(&m3u_path).map_err(|err| {
error!("Could not lock document {:?}: {}", m3u_path, err);
Error::new(ErrorKind::Other, format!("Document Reader error for target {}", &target.name))
}).ok()?;
match IndexedDocumentReader::<M3uPlaylistItem>::new(&m3u_path) {
Ok(mut reader) => {
let server_info = get_user_server_info(cfg, user);
let url = format!("{}/m3u-stream/{}/{}", server_info.get_base_url(), user.username, user.password);
let mut result = vec![];
result.push("#EXTM3U".to_string());
for m3u_pli in reader.by_ref() {
match user.proxy {
ProxyType::Reverse => {
result.push(m3u_pli.to_m3u(target, Some(format!("{url}/{}", m3u_pli.virtual_id).as_str())));
}
ProxyType::Redirect => {
result.push(m3u_pli.to_m3u(target, None));
}
}
};
if reader.by_ref().has_error() {
error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap());
} else {
return Some(result.join("\n"));
}
}
ProxyType::Redirect => {
result.push(m3u_pli.to_m3u(target, None));
Err(err) => {
error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err);
}
}
};
if reader.by_ref().has_error() {
error!("Could not deserialize m3u item {}", &m3u_path.to_str().unwrap());
} else {
return Some(result.join("\n"));
}
}
Err(err) => {
error!("Could not deserialize file {} - {}", &m3u_path.to_str().unwrap(), err);
} else {
error!("Could not open files for target {}", &target.name);
}
}
} else {
error!("Could not open files for target {}", &target.name);
Err(err) => {
error!("Could not find storage path for target {} - {}", target.name.as_str(), err);
}
}
None
}
pub(crate) fn m3u_get_item_for_stream_id(stream_id: u32, m3u_path: &Path, idx_path: &Path) -> Result<M3uPlaylistItem, Error> {
pub(crate) fn m3u_get_item_for_stream_id(cfg: &Config, stream_id: u32, m3u_path: &Path, idx_path: &Path) -> Result<M3uPlaylistItem, Error> {
if stream_id < 1 {
return Err(Error::new(ErrorKind::Other, "id should start with 1"));
}
read_indexed_item::<M3uPlaylistItem>(m3u_path, idx_path, IndexRecord::get_index_offset(stream_id - 1))
{
let _file_lock = cfg.file_locks.read_lock(&m3u_path)?;
IndexedDocumentReader::<M3uPlaylistItem>::read_indexed_item(m3u_path, idx_path, stream_id)
}
}
src/repository/mod.rs
+5 -5
View File
@@ -2,10 +2,10 @@ pub(crate) mod playlist_repository;
pub(crate) mod m3u_repository;
pub(crate) mod xtream_repository;
pub(crate) mod epg_repository;
pub(crate)mod kodi_repository;
pub(crate) mod kodi_repository;
pub(crate) mod storage;
mod bplustree;
mod index_record;
mod indexed_document_writer;
mod indexed_document_reader;
mod indexed_document_reader;
pub(crate) mod target_id_mapping;
pub(crate) mod bplustree;
src/repository/playlist_repository.rs
+58 -14
View File
@@ -1,32 +1,76 @@
use crate::m3u_filter_error::M3uFilterError;
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::model::config::{Config, ConfigTarget, TargetType};
use crate::model::playlist::PlaylistGroup;
use crate::model::xmltv::Epg;
use crate::repository::epg_repository::epg_write;
use crate::repository::kodi_repository::kodi_write_strm_playlist;
use crate::repository::m3u_repository::m3u_write_playlist;
use crate::repository::storage::{ensure_target_storage_path, get_target_id_mapping_file};
use crate::repository::target_id_mapping::TargetIdMapping;
use crate::repository::xtream_repository::xtream_write_playlist;
pub(crate) fn persist_playlist(playlist: &mut [PlaylistGroup], epg: Option<&Epg>,
target: &ConfigTarget, cfg: &Config) -> Result<(), Vec<M3uFilterError>> {
let mut errors = vec![];
for output in &target.output {
match match output.target {
TargetType::M3u => m3u_write_playlist(target, cfg, playlist),
TargetType::Xtream => xtream_write_playlist(target, cfg, playlist),
TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename),
} {
Ok(()) => {
if !playlist.is_empty() {
match epg_write(target, cfg, epg, output) {
Ok(()) => {}
Err(err) => errors.push(err)
match ensure_target_storage_path(cfg, target.name.as_str()) {
Ok(target_path) => {
{
let target_id_mapping_file = get_target_id_mapping_file(&target_path);
match cfg.file_locks.write_lock(&target_id_mapping_file) {
Ok(_file_lock) => {
// reassign old virtual id or assign a new one
let mut target_id_mapping = TargetIdMapping::new(&target_id_mapping_file);
for group in &mut *playlist {
for channel in &group.channels {
let mut header = channel.header.borrow_mut();
match header.get_provider_id() {
Some(provider_id) => {
let uuid = header.get_uuid();
let item_type = header.item_type;
header.virtual_id = target_id_mapping.insert_entry(**uuid, provider_id, &item_type, 0);
}
None => {
errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, format!("Playlistitem has no provider id: {}", &header.title)));
}
}
}
}
for output in &target.output {
match match output.target {
TargetType::M3u => m3u_write_playlist(target, cfg, &target_path, playlist),
TargetType::Xtream => xtream_write_playlist(target, cfg, playlist),
TargetType::Strm => kodi_write_strm_playlist(target, cfg, playlist, &output.filename),
} {
Ok(()) => {
if let Err(err) = target_id_mapping.persist() {
errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()));
}
if !playlist.is_empty() {
match epg_write(target, cfg, &target_path, epg, output) {
Ok(()) => {}
Err(err) => errors.push(err)
}
}
}
Err(err) => errors.push(err)
}
}
if let Err(err) = target_id_mapping.persist() {
errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()));
}
}
Err(err) => {
errors.push(M3uFilterError::new(M3uFilterErrorKind::Info, err.to_string()));
}
}
}
Err(err) => errors.push(err)
}
}
Err(err) => {
errors.push(err);
}
};
if errors.is_empty() { Ok(()) } else { Err(errors) }
}
src/repository/storage.rs
+30
View File
@@ -0,0 +1,30 @@
use std::path::{Path, PathBuf};
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::model::config::Config;
use crate::utils::file_utils;
pub(crate) fn hash_string(url: &str) -> [u8; 32] {
let hash = blake3::hash(url.as_bytes());
hash.into() // Konvertiere den Hash in ein Array mit fester Größe
}
pub(in crate::repository) fn get_target_id_mapping_file(target_path: &Path) -> PathBuf {
target_path.join(PathBuf::from("id_mapping.db"))
}
pub(crate) fn ensure_target_storage_path(cfg: &Config, target_name: &str) -> Result<PathBuf, M3uFilterError> {
if let Some(path) = get_target_storage_path(cfg, target_name) {
if std::fs::create_dir_all(&path).is_err() {
let msg = format!("Failed to save target data, can't create directory {}", &path.to_str().unwrap());
return Err(M3uFilterError::new(M3uFilterErrorKind::Notify, msg));
}
Ok(path)
} else {
let msg = format!("Failed to save target data, can't create directory for target {target_name}");
Err(M3uFilterError::new(M3uFilterErrorKind::Notify, msg))
}
}
pub(crate) fn get_target_storage_path(cfg: &Config, target_name: &str) -> Option<PathBuf> {
file_utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(' ', "_"))))
}
src/repository/target_id_mapping.rs
+107
View File
@@ -0,0 +1,107 @@
use std::cmp::max;
use std::collections::BTreeMap;
use std::io::Error;
use std::path::{Path, PathBuf};
use chrono::Local;
use log::error;
use serde::{Deserialize, Serialize};
use crate::model::playlist::PlaylistItemType;
use crate::repository::bplustree::BPlusTree;
const EXPIRATION_DURATION: i64 = 86400;
#[derive(Serialize, Deserialize, Clone)]
pub(crate) struct VirtualIdRecord {
pub virtual_id: u32,
pub provider_id: u32,
pub uuid: [u8; 32],
pub item_type: PlaylistItemType,
pub parent_virtual_id: u32, // only for series to hold series info id.
pub last_updated: i64,
}
impl VirtualIdRecord {
fn new(provider_id: u32, virtual_id: u32, item_type: PlaylistItemType, parent_virtual_id: u32, uuid: [u8; 32]) -> Self {
let last_updated = Local::now().timestamp();
VirtualIdRecord { provider_id, virtual_id, uuid, item_type, parent_virtual_id, last_updated }
}
pub(crate) fn is_expired(&self) -> bool {
(Local::now().timestamp() - self.last_updated) > EXPIRATION_DURATION
}
pub(crate) fn copy_update_timestamp(&self) -> Self {
Self::new(self.provider_id, self.virtual_id, self.item_type, self.parent_virtual_id, self.uuid)
}
}
pub(crate) struct TargetIdMapping {
dirty: bool,
virtual_id_counter: u32,
by_virtual_id: BPlusTree<u32, VirtualIdRecord>,
by_uuid: BTreeMap<[u8; 32], u32>,
path: PathBuf,
}
impl TargetIdMapping {
pub(crate) fn new(path: &Path) -> Self {
let tree_virtual_id: BPlusTree<u32, VirtualIdRecord> = match BPlusTree::<u32, VirtualIdRecord>::deserialize(&path) {
Ok(tree) => tree,
_ => BPlusTree::<u32, VirtualIdRecord>::new()
};
let mut tree_uuid = BTreeMap::new();
let mut virtual_id_counter: u32 = 0;
tree_virtual_id.traverse(|keys, values| {
match keys.iter().max() {
None => {}
Some(max_value) => {
virtual_id_counter = max(virtual_id_counter, *max_value);
}
}
values.iter().for_each(|v| {
tree_uuid.insert(v.uuid, v.virtual_id);
});
});
TargetIdMapping {
dirty: false,
virtual_id_counter,
by_virtual_id: tree_virtual_id,
by_uuid: tree_uuid,
path: path.to_path_buf(),
}
}
pub(crate) fn insert_entry(&mut self, uuid: [u8; 32], provider_id: u32, item_type: &PlaylistItemType, parent_virtual_id: u32) -> u32 {
match self.by_uuid.get(&uuid) {
None => {
self.dirty = true;
self.virtual_id_counter += 1;
let record = VirtualIdRecord::new(provider_id, self.virtual_id_counter, *item_type, parent_virtual_id, uuid);
self.by_virtual_id.insert(self.virtual_id_counter, record);
self.virtual_id_counter
}
Some(record) => *record
}
}
pub(crate) fn persist(&mut self) -> Result<(), Error> {
if self.dirty {
self.by_virtual_id.serialize(&self.path)?;
}
self.dirty = false;
Ok(())
}
}
impl Drop for TargetIdMapping {
fn drop(&mut self) {
match self.persist() {
Ok(_) => {}
Err(err) => {
error!("Failed to persist target id mapping {:?} err:{}", &self.path, err.to_string())
}
}
}
}
src/repository/xtream_repository.rs
+242 -345
View File
@@ -1,35 +1,30 @@
use std::collections::HashMap;
use std::convert::TryInto;
use std::fs::File;
use std::io::{BufReader, Error, ErrorKind, Read, Seek, SeekFrom, Write};
use std::io::{BufReader, Error, ErrorKind};
use std::path::{Path, PathBuf};
use std::rc::Rc;
use log::error;
use serde_json::{json, Value};
use crate::create_m3u_filter_error_result;
use crate::{create_m3u_filter_error, create_m3u_filter_error_result};
use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind};
use crate::model::config::{Config, ConfigTarget};
use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster, XtreamPlaylistItem};
use crate::model::xtream::XtreamMappingOptions;
use crate::processing::m3u_parser::extract_id_from_url;
use crate::repository::index_record::IndexRecord;
use crate::repository::indexed_document_reader::{IndexedDocumentReader, read_indexed_item};
use crate::repository::bplustree::{BPlusTreeQuery, BPlusTreeUpdate};
use crate::repository::indexed_document_reader::IndexedDocumentReader;
use crate::repository::indexed_document_writer::IndexedDocumentWriter;
use crate::utils::file_utils;
use crate::repository::storage::{get_target_id_mapping_file, get_target_storage_path, hash_string};
use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord};
use crate::utils::json_utils::{json_iter_array, json_write_documents_to_file};
pub(crate) static COL_CAT_LIVE: &str = "cat_live";
pub(crate) static COL_CAT_SERIES: &str = "cat_series";
pub(crate) static COL_CAT_VOD: &str = "cat_vod";
pub(crate) static COL_LIVE: &str = "live";
pub(crate) static COL_SERIES: &str = "series";
pub(crate) static COL_VOD: &str = "vod";
macro_rules! cant_write_result {
($path:expr, $err:expr) => {
create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "failed to write xtream playlist: {} - {}", $path.to_str().unwrap() ,$err)
create_m3u_filter_error!(M3uFilterErrorKind::Notify, "failed to write xtream playlist: {} - {}", $path.to_str().unwrap() ,$err)
}
}
@@ -37,27 +32,6 @@ fn get_collection_path(path: &Path, collection: &str) -> PathBuf {
path.join(format!("{collection}.json"))
}
pub(crate) fn xtream_get_stream_id_cluster_index_file_path(storage_path: &Path) -> PathBuf {
storage_path.join("cluster_index.db")
}
// Maps episode id -> to provider_episode_id and series_id
fn xtream_get_series_episode_id_mapping_file_path(storage_path: &Path) -> PathBuf {
storage_path.join("mapping_episode.db")
}
// maps series_id to the index of series_info index
// direct access is not possible because the series_id is not ascending, it is random
fn xtream_get_series_id_series_info_mapping_file_path(storage_path: &Path) -> PathBuf {
storage_path.join("mapping_series_info.db_idx")
}
// maps catchup_id to provider_id
// direct access is not possible because the series_id is not ascending, it is random
fn xtream_get_catchup_id_mapping_file_path(storage_path: &Path) -> PathBuf {
storage_path.join("mapping_catchup.db")
}
fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result<PathBuf, M3uFilterError> {
if let Some(path) = xtream_get_storage_path(cfg, target_name) {
if std::fs::create_dir_all(&path).is_err() {
@@ -73,7 +47,7 @@ fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result<PathBuf
fn xtream_get_info_file_paths(storage_path: &Path, cluster: XtreamCluster) -> Option<(PathBuf, PathBuf)> {
if cluster == XtreamCluster::Series {
let xtream_path = storage_path.join("series_info.db");
let xtream_path = storage_path.join("series_episodes.db");
let extension = xtream_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or("")));
let index_path = xtream_path.with_extension(format!("{}idx", &extension.unwrap_or_default()));
return Some((xtream_path, index_path));
@@ -81,80 +55,33 @@ fn xtream_get_info_file_paths(storage_path: &Path, cluster: XtreamCluster) -> Op
None
}
fn xtream_clear_series_info(storage_path: &Path) {
if let Some((info_path, idx_path)) = xtream_get_info_file_paths(storage_path, XtreamCluster::Series) {
let _ = std::fs::remove_file(info_path);
let _ = std::fs::remove_file(idx_path);
let _ = std::fs::remove_file(xtream_get_series_episode_id_mapping_file_path(storage_path));
let _ = std::fs::remove_file(xtream_get_series_id_series_info_mapping_file_path(storage_path));
}
}
fn xtream_clear_catchup(storage_path: &Path) {
let _ = std::fs::remove_file(xtream_get_catchup_id_mapping_file_path(storage_path));
}
fn write_playlist_to_file(storage_path: &Path, stream_id: &mut u32, cluster: XtreamCluster, playlist: &mut [PlaylistItem]) -> Result<(), M3uFilterError> {
let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster);
match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) {
Ok(mut writer) => {
for pli in playlist.iter_mut() {
if let Ok(mut xtream) = pli.to_xtream() {
xtream.stream_id = *stream_id;
match writer.write_doc(&xtream) {
Ok(_) => *stream_id += 1,
Err(err) => return cant_write_result!(&xtream_path, err)
}
}
}
if cluster == XtreamCluster::Live {
xtream_clear_catchup(storage_path);
} else if cluster == XtreamCluster::Series {
xtream_clear_series_info(storage_path);
}
Ok(())
}
Err(err) => cant_write_result!(&xtream_path, err)
}
}
fn save_stream_id_cluster_mapping(storage_path: &Path, id_data: &mut Vec<(XtreamCluster, u32, u32)>) -> Result<(), Error> {
let stream_id_path = xtream_get_stream_id_cluster_index_file_path(storage_path);
id_data.sort_by(|(_, start_a, _), (_, start_b, _)| start_b.cmp(start_a));
let encoded: Vec<u8> = bincode::serialize(id_data).unwrap();
std::fs::write(stream_id_path, encoded)
}
fn load_stream_id_cluster_mapping(storage_path: &Path) -> Option<Vec<(XtreamCluster, u32, u32)>> {
let path = xtream_get_stream_id_cluster_index_file_path(storage_path);
if path.exists() {
match std::fs::read(path) {
Ok(encoded) => {
let decoded: Vec<(XtreamCluster, u32, u32)> = bincode::deserialize(&encoded[..]).unwrap();
Some(decoded)
}
Err(_) => None,
}
} else {
None
}
}
fn write_playlists_to_file(storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> {
let mut id_list: Vec<(XtreamCluster, u32, u32)> = vec![];
let mut stream_id: u32 = 1;
fn write_playlists_to_file(cfg: &Config, storage_path: &Path, collections: Vec<(XtreamCluster, &mut [PlaylistItem])>) -> Result<(), M3uFilterError> {
for (cluster, playlist) in collections {
let start = stream_id;
write_playlist_to_file(storage_path, &mut stream_id, cluster, playlist)?;
id_list.push((cluster, start, stream_id));
}
match save_stream_id_cluster_mapping(storage_path, &mut id_list) {
Ok(()) => Ok(()),
Err(err) => Err(M3uFilterError::new(M3uFilterErrorKind::Notify, format!("failed to write xtream playlist: {} - {}", storage_path.to_str().unwrap(), err)))
let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, &cluster);
{
let _file_lock = cfg.file_locks.write_lock(&xtream_path).map_err(|err| M3uFilterError::new(M3uFilterErrorKind::Info, format!("{}", err)))?;
match IndexedDocumentWriter::new(xtream_path.clone(), idx_path) {
Ok(mut writer) => {
for item in playlist {
match item.to_xtream() {
Ok(xtream) => {
match writer.write_doc(item.header.borrow().virtual_id, &xtream) {
Ok(_) => {}
Err(err) => return Err(cant_write_result!(&xtream_path, err))
}
}
Err(err) => return Err(cant_write_result!(&xtream_path, err))
}
}
writer.flush().map_err(|err| cant_write_result!(&xtream_path, err))?;
}
Err(err) => return Err(cant_write_result!(&xtream_path, err))
}
}
}
Ok(())
}
fn get_map_item_as_str(map: &serde_json::Map<String, Value>, key: &str) -> Option<String> {
if let Some(value) = map.get(key) {
if let Some(result) = value.as_str() {
@@ -193,26 +120,33 @@ fn load_old_category_ids(path: &Path) -> (u32, HashMap<String, u32>) {
}
pub(crate) fn xtream_get_storage_path(cfg: &Config, target_name: &str) -> Option<PathBuf> {
file_utils::get_file_path(&cfg.working_dir, Some(std::path::PathBuf::from(target_name.replace(' ', "_"))))
match get_target_storage_path(cfg, target_name) {
Some(target_path) => Some(target_path.join(std::path::PathBuf::from("xtream"))),
None => None,
}
}
pub(crate) fn xtream_get_epg_file_path(path: &Path) -> PathBuf {
path.join("epg.xml")
}
pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: XtreamCluster) -> (PathBuf, PathBuf) {
let xtream_path = storage_path.join(format!("{}.db", match cluster {
XtreamCluster::Live => COL_LIVE,
XtreamCluster::Video => COL_VOD,
XtreamCluster::Series => COL_SERIES
}));
fn xtream_get_file_paths_for_name(storage_path: &Path, name: &str) -> (PathBuf, PathBuf) {
let xtream_path = storage_path.join(format!("{}.db", name));
let extension = xtream_path.extension().map(|ext| format!("{}_", ext.to_str().unwrap_or("")));
let index_path = xtream_path.with_extension(format!("{}idx", &extension.unwrap_or_default()));
(xtream_path, index_path)
}
pub(crate) fn xtream_get_file_paths(storage_path: &Path, cluster: &XtreamCluster) -> (PathBuf, PathBuf) {
xtream_get_file_paths_for_name(storage_path, &cluster.as_str().to_lowercase())
}
pub(crate) fn xtream_get_file_paths_for_series(storage_path: &Path) -> (PathBuf, PathBuf) {
xtream_get_file_paths_for_name(storage_path, "series")
}
pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlist: &mut [PlaylistGroup]) -> Result<(), M3uFilterError> {
match ensure_xtream_storage_path(cfg, target.name.replace(' ', "_").as_str()) {
match ensure_xtream_storage_path(cfg, target.name.as_str()) {
Ok(path) => {
let mut cat_live_col = vec![];
let mut cat_series_col = vec![];
@@ -249,30 +183,8 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis
let col = if header.item_type == PlaylistItemType::Series {
None
} else {
if header.id.parse::<i32>().is_err() {
let id_from_url = match extract_id_from_url(&header.url) {
Some(id) => match id.parse::<i32>() {
Ok(newid) => Some(newid),
Err(_) => None,
},
None => None,
};
let has_id = match id_from_url {
Some(newid) => {
header.id = Rc::new(newid.to_string());
Ok(())
}
None => {
let title = header.title.as_str();
errors.push(format!("Channel does not have an id: {title}"));
Err(())
}
};
// Instead of returning from the function, handle the error by assigning None to col.
if has_id.is_err() {
None
} else {
match header.get_provider_id() {
Some(_) => {
header.category_id = *cat_id;
Some(match header.xtream_cluster {
XtreamCluster::Live => &mut live_col,
@@ -280,13 +192,11 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis
XtreamCluster::Video => &mut vod_col,
})
}
} else {
header.category_id = *cat_id;
Some(match header.xtream_cluster {
XtreamCluster::Live => &mut live_col,
XtreamCluster::Series => &mut series_col,
XtreamCluster::Video => &mut vod_col,
})
None => {
let title = header.title.as_str();
errors.push(format!("Channel does not have an id: {title}"));
None
}
}
};
drop(header);
@@ -309,7 +219,7 @@ pub(crate) fn xtream_write_playlist(target: &ConfigTarget, cfg: &Config, playlis
}
}
match write_playlists_to_file(&path, vec![
match write_playlists_to_file(cfg, &path, vec![
(XtreamCluster::Live, &mut live_col),
(XtreamCluster::Video, &mut vod_col),
(XtreamCluster::Series, &mut series_col)]) {
@@ -338,235 +248,222 @@ pub(crate) fn xtream_get_collection_path(cfg: &Config, target_name: &str, collec
Err(Error::new(ErrorKind::Other, format!("Cant find collection: {target_name}/{collection_name}")))
}
fn _xtream_get_item_for_stream_id(stream_id: u32, storage_path: &Path, xtream_cluster: Option<XtreamCluster>, mapping: &[(XtreamCluster, u32, u32)]) -> Result<XtreamPlaylistItem, Error> {
if let Some((cluster, cluster_start, _end)) = match xtream_cluster {
Some(clus) => mapping.iter().find(|(c, _, _)| *c == clus),
None => mapping.iter().find(|(_cluster, start, _end)| stream_id >= *start),
} {
let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, *cluster);
if stream_id >= *cluster_start {
return read_indexed_item::<XtreamPlaylistItem>(&xtream_path, &idx_path, IndexRecord::get_index_offset(stream_id - cluster_start));
}
fn xtream_read_item_for_stream_id(cfg: &Config, stream_id: u32, storage_path: &Path, cluster: &XtreamCluster) -> Result<XtreamPlaylistItem, Error> {
let (xtream_path, idx_path) = xtream_get_file_paths(storage_path, cluster);
{
let _file_lock = cfg.file_locks.read_lock(&xtream_path)?;
IndexedDocumentReader::<XtreamPlaylistItem>::read_indexed_item(&xtream_path, &idx_path, stream_id)
}
Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}")))
}
pub(crate) fn xtream_get_item_for_stream_id(stream_id: u32, config: &Config, target: &ConfigTarget, xtream_cluster: Option<XtreamCluster>) -> Result<XtreamPlaylistItem, Error> {
if let Some(storage_path) = xtream_get_storage_path(config, target.name.replace(' ', "_").as_str()) {
if let Some(mapping) = load_stream_id_cluster_mapping(&storage_path) {
if let Some(max) = mapping.iter().map(|(_cluster, _start, end)| end).max().copied() {
if max < stream_id {
// episoden id's fangen bei (max cluster id + 1) an.
// episode mapping har 3 u32 also 12 bytes
let index = stream_id - (max + 1);
if let Ok((_episode_id, provider_id, series_id)) = xtream_read_episode_id_mapping(&storage_path, index) {
if let Ok(mut pli) = _xtream_get_item_for_stream_id(series_id, &storage_path, xtream_cluster, &mapping) {
pli.provider_id = provider_id;
return Ok(pli);
}
fn xtream_read_series_item_for_stream_id(cfg: &Config, stream_id: u32, storage_path: &Path) -> Result<XtreamPlaylistItem, Error> {
let (xtream_path, idx_path) = xtream_get_file_paths_for_series(storage_path);
{
let _file_lock = cfg.file_locks.read_lock(&xtream_path)?;
return IndexedDocumentReader::<XtreamPlaylistItem>::read_indexed_item(&xtream_path, &idx_path, stream_id);
}
}
macro_rules! try_cluster {
($xtream_cluster:expr, $item_type:expr, $virtual_id:expr) => {
$xtream_cluster.or_else(|| XtreamCluster::try_from($item_type).ok())
.ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not determine cluster for xtream item with stream-id {}", $virtual_id)))
};
}
pub(crate) fn xtream_get_item_for_stream_id(
virtual_id: u32,
config: &Config,
target: &ConfigTarget,
xtream_cluster: Option<XtreamCluster>,
) -> Result<XtreamPlaylistItem, Error> {
let target_path = get_target_storage_path(config, target.name.as_str())
.ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {}", &target.name)))?;
let storage_path = xtream_get_storage_path(config, target.name.as_str())
.ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {} xtream output", &target.name)))?;
{
let target_id_mapping_file = get_target_id_mapping_file(&target_path);
let _file_lock = config.file_locks.read_lock(&target_id_mapping_file)
.map_err(|err| Error::new(ErrorKind::Other, format!("Could not get lock for id mapping for target {} err:{}", target.name, err.to_string())))?;
let mut target_id_mapping = BPlusTreeQuery::<u32, VirtualIdRecord>::try_new(&target_id_mapping_file)
.map_err(|err| Error::new(ErrorKind::Other, format!("Could not load id mapping for target {} err:{}", target.name, err.to_string())))?;
let mapping = target_id_mapping
.query(&virtual_id)
.ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find mapping for target {} and id {}", target.name, virtual_id)))?;
match mapping.item_type {
PlaylistItemType::SeriesInfo => xtream_read_series_item_for_stream_id(config, virtual_id, &storage_path),
PlaylistItemType::SeriesEpisode => {
let mut item = xtream_read_series_item_for_stream_id(config, mapping.parent_virtual_id, &storage_path)?;
item.provider_id = mapping.provider_id;
Ok(item)
}
PlaylistItemType::Catchup => {
let cluster = try_cluster!(xtream_cluster, mapping.item_type, virtual_id)?;
let mut item = xtream_read_item_for_stream_id(config, mapping.parent_virtual_id, &storage_path, &cluster)?;
item.provider_id = mapping.provider_id;
Ok(item)
}
_ => {
let cluster = try_cluster!(xtream_cluster, mapping.item_type, virtual_id)?;
xtream_read_item_for_stream_id(config, virtual_id, &storage_path, &cluster)
}
}
}
}
pub(crate) fn xtream_load_rewrite_playlist(cluster: &XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result<String, Error> {
if let Some(storage_path) = xtream_get_storage_path(config, target.name.as_str()) {
let (xtream_path, _) = xtream_get_file_paths(&storage_path, cluster);
{
let _file_lock = config.file_locks.read_lock(&xtream_path)?;
match IndexedDocumentReader::<XtreamPlaylistItem>::new(&xtream_path) {
Ok(mut reader) => {
let options = XtreamMappingOptions::from_target_options(target.options.as_ref());
let result: Vec<Value> = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id)
.map(|pli| pli.to_doc(&options)).collect();
if reader.by_ref().has_error() {
error!("Could not deserialize item {}", &xtream_path.to_str().unwrap());
} else {
return Ok(serde_json::to_string(&result).unwrap());
}
return Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}")));
}
}
return _xtream_get_item_for_stream_id(stream_id, &storage_path, xtream_cluster, &mapping);
}
}
Err(Error::new(ErrorKind::Other, format!("Failed to read xtream item for stream-id {stream_id}")))
}
pub(crate) fn xtream_load_rewrite_playlist(cluster: XtreamCluster, config: &Config, target: &ConfigTarget, category_id: u32) -> Result<String, Error> {
if let Some(storage_path) = xtream_get_storage_path(config, target.name.replace(' ', "_").as_str()) {
let (xtream_path, idx_path) = xtream_get_file_paths(&storage_path, cluster);
match IndexedDocumentReader::<XtreamPlaylistItem>::new(&xtream_path, &idx_path) {
Ok(mut reader) => {
let options = XtreamMappingOptions::from_target_options(target.options.as_ref());
let result: Vec<Value> = reader.by_ref().filter(|pli| category_id == 0 || pli.category_id == category_id)
.map(|pli| pli.to_doc(&options)).collect();
if reader.by_ref().has_error() {
error!("Could not deserialize item {}", &xtream_path.to_str().unwrap());
} else {
return Ok(serde_json::to_string(&result).unwrap());
Err(err) => {
error!("Could not deserialize file {} - {}", &xtream_path.to_str().unwrap(), err);
}
}
Err(err) => {
error!("Could not deserialize file {} - {}", &xtream_path.to_str().unwrap(), err);
}
}
}
Err(Error::new(ErrorKind::Other, format!("Failed to find xtream storage for target {}", &target.name)))
}
// The mapping file record is episode_id 4bytes, episode_provider_id 4bytes, series_id 4bytes
fn read_last_id_from_episode_id_mapping_file(path: &Path) -> Result<u32, Error> {
let mut file = File::open(path)?;
file.seek(SeekFrom::End(-12))?;
let mut buffer = [0; 4];
file.read_exact(&mut buffer)?;
Ok(u32::from_le_bytes(buffer))
macro_rules! try_option_ok {
($option:expr) => {
match $option {
Some(value) => value,
None => return Ok(()),
}
};
}
pub(crate) fn xtream_get_max_series_info_episode_id(config: &Config, target_name: &str) -> Option<u32> {
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
let mapping_file_path = xtream_get_series_episode_id_mapping_file_path(&storage_path);
if mapping_file_path.exists() {
if let Ok(last_id) = read_last_id_from_episode_id_mapping_file(&mapping_file_path) {
return Some(last_id);
pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str,
series_info_id: u32,
content: &str) -> Result<(), Error> {
let target_path = try_option_ok!(get_target_storage_path(config, target_name));
let storage_path = try_option_ok!(xtream_get_storage_path(config, target_name));
let (info_path, idx_path) = try_option_ok!(xtream_get_info_file_paths(&storage_path, XtreamCluster::Series));
{
let _file_lock = config.file_locks.write_lock(&info_path)?;
let mut writer = IndexedDocumentWriter::new_append(info_path.clone(), idx_path)?;
writer
.write_doc(series_info_id, content)
.map_err(|_| Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}")))?;
writer.flush()?;
}
{
let target_id_mapping_file = get_target_id_mapping_file(&target_path);
let _file_lock = config.file_locks.write_lock(&target_id_mapping_file)?;
if let Ok(mut target_id_mapping) = BPlusTreeUpdate::<u32, VirtualIdRecord>::try_new(&target_id_mapping_file) {
if let Some(record) = target_id_mapping.query(&series_info_id) {
let new_record = record.copy_update_timestamp();
let _ = target_id_mapping.update(&series_info_id, new_record);
}
};
}
Ok(())
}
// Reads the series info entry if exists, otherwise error
pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Option<String> {
let target_path = get_target_storage_path(config, target_name)?;
let storage_path = xtream_get_storage_path(config, target_name)?;
{
let target_id_mapping_file = get_target_id_mapping_file(&target_path);
let _file_lock = config.file_locks.read_lock(&target_id_mapping_file).map_err(|err| {
error!("Could not lock id mapping for target {target_name}: {}", err);
Error::new(ErrorKind::Other, format!("ID mapping load error for target {target_name}"))
}).ok()?;
let mut target_id_mapping = BPlusTreeQuery::<u32, VirtualIdRecord>::try_new(&target_id_mapping_file)
.map_err(|err| {
error!("Could not load id mapping for target {target_name}: {}", err);
Error::new(ErrorKind::Other, format!("ID mapping load error for target {target_name}"))
}).ok()?;
if let Some(id_record) = target_id_mapping.query(&series_id) {
if id_record.is_expired() {
return None;
}
}
}
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
if let Some(mapping) = load_stream_id_cluster_mapping(&storage_path) {
return mapping.iter().map(|(_, _, end)| end).max().copied();
}
let (info_path, idx_path) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)?;
if info_path.exists() && idx_path.exists() {
{
let _file_lock = config.file_locks.read_lock(&info_path).map_err(|err| {
error!("Could not lock document {:?}: {}", info_path, err);
Error::new(ErrorKind::Other, format!("Document Reader error for target {target_name}"))
}).ok()?;
return match IndexedDocumentReader::<String>::read_indexed_item(&info_path, &idx_path, series_id) {
Ok(content) => Some(content),
Err(err) => {
error!("Failed to read series info for id {series_id} for {target_name}: {}", err);
None
}
};
}
}
None
}
fn xtream_write_episode_id_mapping(storage_path: &Path, series_id: u32, episode_id_mapping: &[(u32, u32)]) -> Result<(), Error> {
let file_path = xtream_get_series_episode_id_mapping_file_path(storage_path);
if let Ok(mut file) = if file_path.exists() {
std::fs::OpenOptions::new()
.append(true) // Open in append mode
.open(file_path)
} else {
File::create(file_path)
} {
let stream_id_bytes: [u8; 4] = series_id.to_le_bytes();
let mut bytes: Vec<u8> = Vec::new();
for (episode_id, provider_id) in episode_id_mapping {
bytes.extend_from_slice(&episode_id.to_le_bytes());
bytes.extend_from_slice(&provider_id.to_le_bytes());
bytes.extend_from_slice(&stream_id_bytes);
}
return file_utils::check_write(&file.write_all(&bytes[..]));
}
Err(Error::new(ErrorKind::Other, format!("Failed to open series info idmapping file inside {}", storage_path.to_str().unwrap())))
}
pub(crate) fn write_and_get_xtream_series_info(
config: &Config,
target: &ConfigTarget,
pli_series_info: &XtreamPlaylistItem,
content: &str,
) -> Result<String, Error> {
let mut doc = serde_json::from_str::<Value>(content)
.map_err(|_| Error::new(ErrorKind::Other, "Failed to parse JSON content"))?;
fn xtream_read_episode_id_mapping(storage_path: &Path, index: u32) -> Result<(u32, u32, u32), Error> {
let offset = index * 12;
let episode_file = xtream_get_series_episode_id_mapping_file_path(storage_path);
if episode_file.exists() {
if let Ok(mut file) = File::open(episode_file) {
file.seek(SeekFrom::Start(u64::from(offset)))?;
let mut bytes = [0u8; 4];
file.read_exact(&mut bytes)?;
let episode_id = u32::from_le_bytes(bytes);
file.read_exact(&mut bytes)?;
let provider_id = u32::from_le_bytes(bytes);
file.read_exact(&mut bytes)?;
let series_id = u32::from_le_bytes(bytes);
Ok((episode_id, provider_id, series_id))
} else {
Err(Error::new(ErrorKind::Other, format!("Could not find episode mapping at offset {offset}")))
}
} else {
Err(Error::new(ErrorKind::Other, format!("Episode mapping not found at {}", storage_path.to_str().unwrap_or(""))))
}
}
let target_path = get_target_storage_path(config, target.name.as_str())
.ok_or_else(|| Error::new(ErrorKind::Other, format!("Could not find path for target {}", target.name)))?;
pub(crate) fn xtream_write_series_info(config: &Config, target_name: &str,
series_id: u32, episode_id_mapping: &[(u32, u32)],
content: &str) -> Result<(), Error> {
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) {
return match IndexedDocumentWriter::new_append(info_path.clone(), idx_path) {
Ok(mut writer) => {
match writer.write_doc(content) {
Ok((_, index_offset)) => {
let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path);
IndexRecord::to_file(&series_id_index_mapping_path, series_id, index_offset, true)?;
}
Err(_) => return Err(Error::new(ErrorKind::Other, format!("failed to write xtream series info for target {target_name}")))
}
xtream_write_episode_id_mapping(&storage_path, series_id, episode_id_mapping)
let episodes = doc.get_mut("episodes")
.and_then(Value::as_object_mut)
.ok_or_else(|| Error::new(ErrorKind::Other, "No episodes found in content"))?;
{
let target_id_mapping_file = get_target_id_mapping_file(&target_path);
let _file_lock = config.file_locks.write_lock(&target_id_mapping_file)
.map_err(|err| Error::new(ErrorKind::Other, format!("Could not load id mapping for target {} err:{}", target.name, err.to_string())))?;
let mut target_id_mapping = TargetIdMapping::new(&target_id_mapping_file);
let options = XtreamMappingOptions::from_target_options(target.options.as_ref());
for episode_list in episodes.values_mut().filter_map(Value::as_array_mut) {
for episode in episode_list.iter_mut().filter_map(Value::as_object_mut) {
if let Some(provider_id) = episode.get("id").and_then(Value::as_str).and_then(|id| id.parse::<u32>().ok()) {
let uuid = hash_string(&format!("{}/{}", pli_series_info.url, provider_id));
let virtual_id = target_id_mapping.insert_entry(uuid, provider_id, &PlaylistItemType::SeriesEpisode, pli_series_info.virtual_id);
episode.insert("id".to_string(), Value::String(virtual_id.to_string()));
}
Err(err) => Err(err)
};
}
}
Ok(())
}
pub(crate) fn xtream_load_series_info(config: &Config, target_name: &str, series_id: u32) -> Result<String, Error> {
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
let series_id_index_mapping_path = xtream_get_series_id_series_info_mapping_file_path(&storage_path);
if series_id_index_mapping_path.exists() {
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
if let Some((info_path, idx_path)) = xtream_get_info_file_paths(&storage_path, XtreamCluster::Series) {
if info_path.exists() && idx_path.exists() {
let mut file = File::open(series_id_index_mapping_path)?;
let mut buffer = [0u8; 8];
loop {
match file.read_exact(&mut buffer) {
Ok(()) => {
let stream_id = u32::from_le_bytes(buffer[..4].try_into().unwrap());
if stream_id == series_id {
let index = u32::from_le_bytes(buffer[4..].try_into().unwrap());
return read_indexed_item::<String>(&info_path, &idx_path, index);
}
}
Err(e) => return Err(e),
}
}
}
if options.skip_series_direct_source {
episode.insert("direct_source".to_string(), Value::String(String::new()));
}
}
}
}
Err(Error::new(ErrorKind::Other, format!("Failed to read series info for id {series_id} for {target_name}")))
}
pub(crate) fn xtream_write_catchup_id_mapping(config: &Config, target_name: &str, id_mappings: &Vec<(u32, u32)>) -> Result<(), Error> {
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
let file_path = xtream_get_catchup_id_mapping_file_path(&storage_path);
if let Ok(mut file) = if file_path.exists() {
std::fs::OpenOptions::new()
.append(true) // Open in append mode
.open(file_path)
} else {
File::create(file_path)
} {
for (provider_id, stream_id) in id_mappings {
let mut bytes: Vec<u8> = Vec::new();
bytes.extend_from_slice(&provider_id.to_le_bytes());
bytes.extend_from_slice(&stream_id.to_le_bytes());
file_utils::check_write(&file.write_all(&bytes[..]))?;
}
return Ok(());
}
drop(target_id_mapping);
}
Err(Error::new(ErrorKind::Other, format!("Failed to get catchup info for {target_name}")))
}
let result = serde_json::to_string(&doc)
.map_err(|_| Error::new(ErrorKind::Other, "Failed to serialize updated series info"))?;
xtream_write_series_info(config, target.name.as_str(), pli_series_info.virtual_id, &result).ok();
// Returns hashmap with provider_id -> new_stream_id
pub(crate) fn xtream_load_catchup_id_mapping(config: &Config, target_name: &str) -> HashMap<u32, u32> {
let mut result = HashMap::new();
if let Some(storage_path) = xtream_get_storage_path(config, target_name) {
let catchup_file = xtream_get_catchup_id_mapping_file_path(&storage_path);
if catchup_file.exists() {
if let Ok(file) = File::open(catchup_file) {
let mut reader = BufReader::new(&file);
let mut bytes = [0u8; 4];
loop {
if reader.read_exact(&mut bytes).is_ok() {
let provider_id = u32::from_le_bytes(bytes);
let mut bytes = [0u8; 4];
if reader.read_exact(&mut bytes).is_ok() {
let stream_id = u32::from_le_bytes(bytes);
result.insert(provider_id, stream_id);
} else {
break;
}
} else {
break;
}
}
}
}
} else {
error!("Failed to open catchup id-mapping file for {target_name}");
}
result
Ok(result)
}
src/test.rs
+2 -20
View File
@@ -4,32 +4,14 @@ mod tests {
use std::rc::Rc;
use regex::Regex;
use crate::filter::{get_filter, MockValueProcessor, ValueProvider};
use crate::model::playlist::{PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
use crate::model::playlist::{PlaylistItem, PlaylistItemHeader};
fn create_mock_pli(name: &str, group: &str) -> PlaylistItem {
PlaylistItem {
header: RefCell::new(PlaylistItemHeader {
uuid: Rc::new("".to_string()),
stream_id: Rc::new("".to_string()),
id: Rc::new("".to_string()),
name: Rc::new(name.to_string()),
chno: Rc::new("".to_string()),
logo: Rc::new("".to_string()),
logo_small: Rc::new("".to_string()),
group: Rc::new(group.to_string()),
title: Rc::new("".to_string()),
parent_code: Rc::new("".to_string()),
audio_track: Rc::new("".to_string()),
time_shift: Rc::new("".to_string()),
rec: Rc::new("".to_string()),
url: Rc::new("".to_string()),
epg_channel_id: None,
xtream_cluster: XtreamCluster::Live,
additional_properties: None,
item_type: PlaylistItemType::Live,
series_fetched: false,
category_id: 0,
input_id: 0,
..Default::default()
})
}
}
src/utils/default_utils.rs
-3
View File
@@ -1,7 +1,6 @@
use std::collections::HashMap;
use std::rc::Rc;
use crate::model::config::ProcessingOrder;
use crate::model::playlist::{PlaylistItemType, XtreamCluster};
pub(crate) fn default_as_true() -> bool { true }
@@ -23,8 +22,6 @@ pub(crate) fn default_as_empty_list<T>() -> Vec<T> { vec![] }
pub(crate) fn default_as_two_u16() -> u16 { 2 }
pub(crate) fn default_playlist_item_type() -> PlaylistItemType { PlaylistItemType::Live }
pub(crate) fn default_as_zero_u32() -> u32 { 0 }
pub(crate) fn default_as_zero_u16() -> u16 { 0 }
pub(crate) fn default_stream_cluster() -> XtreamCluster { XtreamCluster::Live }
src/utils/download.rs
+24 -23
View File
@@ -30,8 +30,7 @@ pub(crate) async fn get_m3u_playlist(cfg: &Config, input: &ConfigInput, working_
let persist_file_path = prepare_file_path(input.persist.as_ref(), working_dir, "");
match request_utils::get_input_text_content(input, working_dir, &url, persist_file_path).await {
Ok(text) => {
let lines = text.lines().map(String::from).collect::<Vec<String>>();
(m3u_parser::parse_m3u(cfg, input, &lines), vec![])
(m3u_parser::parse_m3u(cfg, input, text.lines()), vec![])
}
Err(err) => (vec![], vec![err])
}
@@ -57,7 +56,7 @@ pub(crate) async fn get_xtream_playlist_series<'a>(fpl: &mut FetchedPlaylist<'a>
match parse_xtream_series_info(&series_content, pli.header.borrow().group.as_str(), input) {
Ok(series_info) => {
if let Some(mut series) = series_info {
series.drain(..).for_each(|item| group_series.push(item));
group_series.extend(series.drain(..));
}
}
Err(err) => errors.push(err),
@@ -108,7 +107,7 @@ const ACTIONS: [(XtreamCluster, &str, &str); 3] = [
(XtreamCluster::Series, "get_series_categories", "get_series")];
pub(crate) async fn get_xtream_playlist(input: &ConfigInput, working_dir: &String) -> (Vec<PlaylistGroup>, Vec<M3uFilterError>) {
let mut playlist: Vec<PlaylistGroup> = Vec::new();
let mut playlist_groups: Vec<PlaylistGroup> = Vec::new();
let username = input.username.as_ref().map_or("", |v| v);
let password = input.password.as_ref().map_or("", |v| v);
let base_url = format!("{}/player_api.php?username={}&password={}", input.url, username, password);
@@ -123,35 +122,37 @@ pub(crate) async fn get_xtream_playlist(input: &ConfigInput, working_dir: &Strin
let category_file_path = prepare_file_path(input.persist.as_ref(), working_dir, format!("{category}_").as_str());
let stream_file_path = prepare_file_path(input.persist.as_ref(), working_dir, format!("{stream}_").as_str());
match request_utils::get_input_json_content(input, category_url.as_str(), category_file_path).await {
Ok(category_content) => {
match request_utils::get_input_json_content(input, stream_url.as_str(), stream_file_path).await {
Ok(stream_content) => {
match xtream_parser::parse_xtream(input,
*xtream_cluster,
&category_content,
&stream_content) {
Ok(sub_playlist_opt) => {
if let Some(mut sub_playlist) = sub_playlist_opt {
sub_playlist.drain(..).for_each(|group| playlist.push(group));
}
}
Err(err) => errors.push(err)
match futures::join!(
request_utils::get_input_json_content(input, category_url.as_str(), category_file_path),
request_utils::get_input_json_content(input, stream_url.as_str(), stream_file_path)
) {
(Ok(category_content), Ok(stream_content)) => {
match xtream_parser::parse_xtream(input,
*xtream_cluster,
&category_content,
&stream_content) {
Ok(sub_playlist_parsed) => {
if let Some(mut xtream_sub_playlist) = sub_playlist_parsed {
playlist_groups.extend(xtream_sub_playlist.drain(..));
}
}
Err(err) => errors.push(err)
}
}
Err(err) => errors.push(err)
},
(Err(err1), Err(err2)) => {
errors.extend([err1, err2]);
},
(Err(err), _) => errors.push(err),
(_, Err(err)) => errors.push(err),
}
}
}
playlist.sort_by(|a, b| a.title.partial_cmp(&b.title).unwrap_or(Ordering::Greater));
playlist_groups.sort_by(|a, b| a.title.partial_cmp(&b.title).unwrap_or(Ordering::Greater));
for (grp_id, plg) in (1_u32..).zip(playlist.iter_mut()) {
for (grp_id, plg) in (1_u32..).zip(playlist_groups.iter_mut()) {
plg.id = grp_id;
}
(playlist, errors)
(playlist_groups, errors)
}
pub(crate) async fn get_xmltv(_cfg: &Config, input: &ConfigInput, working_dir: &String) -> (Option<TVGuide>, Vec<M3uFilterError>) {
src/utils/file_lock_manager.rs
+106
View File
@@ -0,0 +1,106 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::{fmt, io};
use std::path::{Path, PathBuf};
#[derive(Clone)]
pub(crate) struct FileLockManager {
locks: Arc<Mutex<HashMap<PathBuf, Arc<RwLock<()>>>>>,
}
impl FileLockManager {
pub(crate) fn new() -> Self {
Self {
locks: Arc::new(Mutex::new(HashMap::new())),
}
}
// Acquires a read lock for the specified file and returns a FileReadGuard.
pub(crate) fn read_lock(&self, path: &Path) -> io::Result<FileReadGuard> {
let file_lock = self.get_or_create_lock(path)?;
let guard = file_lock.read().map_err(|_| {
io::Error::new(io::ErrorKind::Other, "Failed to acquire read lock")
})?;
// Clone the Arc to avoid moving `file_lock` out, as it is still borrowed by `guard`
Ok(FileReadGuard::new(Arc::clone(&file_lock), guard))
}
// Acquires a write lock for the specified file and returns a FileWriteGuard.
pub(crate) fn write_lock(&self, path: &Path) -> io::Result<FileWriteGuard> {
let file_lock = self.get_or_create_lock(path)?;
let guard = file_lock.write().map_err(|_| {
io::Error::new(io::ErrorKind::Other, "Failed to acquire write lock")
})?;
// Clone the Arc to avoid moving `file_lock` out, as it is still borrowed by `guard`
Ok(FileWriteGuard::new(Arc::clone(&file_lock), guard))
}
// Helper function: retrieves or creates a lock for a file.
fn get_or_create_lock(&self, path: &Path) -> io::Result<Arc<RwLock<()>>> {
let mut locks = self.locks.lock().map_err(|_| {
io::Error::new(io::ErrorKind::Other, "Failed to acquire lock on lock manager")
})?;
if let Some(lock) = locks.get(path) {
return Ok(lock.clone());
}
let file_lock = Arc::new(RwLock::new(()));
locks.insert(path.to_path_buf(), file_lock.clone());
Ok(file_lock)
}
}
impl Default for FileLockManager {
fn default() -> Self {
FileLockManager::new()
}
}
impl fmt::Debug for FileLockManager {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Acquire the lock to safely access the HashMap
let locks = self.locks.lock().unwrap();
// Format the paths in the HashMap for debug purposes
let keys: Vec<_> = locks.keys().collect();
f.debug_struct("FileLockManager")
.field("locks", &keys)
.finish()
}
}
// Define FileReadGuard to hold both the lock reference and the actual read guard.
#[allow(dead_code)]
pub(crate) struct FileReadGuard {
lock: Arc<RwLock<()>>,
guard: RwLockReadGuard<'static, ()>,
}
impl FileReadGuard {
pub fn new(lock: Arc<RwLock<()>>, guard: RwLockReadGuard<'_, ()>) -> Self {
// Convert the lifetime of `guard` to 'static by transmuting.
let static_guard: RwLockReadGuard<'static, ()> = unsafe { std::mem::transmute(guard) };
FileReadGuard {
lock,
guard: static_guard,
}
}
}
// Define FileWriteGuard to hold both the lock reference and the actual write guard.
#[allow(dead_code)]
pub(crate) struct FileWriteGuard {
lock: Arc<RwLock<()>>,
guard: RwLockWriteGuard<'static, ()>,
}
impl FileWriteGuard {
pub fn new(lock: Arc<RwLock<()>>, guard: RwLockWriteGuard<'_, ()>) -> Self {
// Convert the lifetime of `guard` to 'static by transmuting.
let static_guard: RwLockWriteGuard<'static, ()> = unsafe { std::mem::transmute(guard) };
FileWriteGuard {
lock,
guard: static_guard,
}
}
}
src/utils/file_utils.rs
-13
View File
@@ -196,16 +196,3 @@ pub(crate) fn open_file_append(path: &Path, append: bool) -> Result<File, std::i
File::create(path)
}
pub(crate) fn create_file_tuple(path1: &Path, path2: &Path, append: bool) -> Result<(File, File), std::io::Error> {
match open_file_append(path1, append) {
Ok(file1) => {
match open_file_append(path2, append) {
Ok(file2) => Ok((file1, file2)),
Err(err) =>
Err(std::io::Error::new(err.kind(), format!("failed to create file {} - {}", path2.to_str().unwrap(), err)))
}
}
Err(err) =>
Err(std::io::Error::new(err.kind(), format!("failed to create file {} - {}", path1.to_str().unwrap(), err)))
}
}
src/utils/mod.rs
+1 -1
View File
@@ -6,4 +6,4 @@ pub (crate) mod json_utils;
pub (crate) mod config_reader;
pub (crate) mod default_utils;
pub (crate) mod multi_file_reader;
// pub (crate) mod file_reader;
pub (crate) mod file_lock_manager;
src/utils/request_utils.rs
+12 -12
View File
@@ -232,15 +232,15 @@ pub(crate) async fn get_input_json_content(input: &ConfigInput, url: &str, persi
Err(e) => create_m3u_filter_error_result!(M3uFilterErrorKind::Notify, "cant download input url: {url} => {}", e)
}
}
pub(crate) fn get_base_url(url: &str) -> Option<String> {
if let Some((scheme_end, rest)) = url.split_once("://") {
let scheme = scheme_end;
if let Some(authority_end) = rest.find('/') {
let authority = &rest[..authority_end];
return Some(format!("{}://{}", scheme, authority));
}
return Some(format!("{}://{}", scheme, rest));
}
None
}
//
// pub(crate) fn get_base_url(url: &str) -> Option<String> {
// if let Some((scheme_end, rest)) = url.split_once("://") {
// let scheme = scheme_end;
// if let Some(authority_end) = rest.find('/') {
// let authority = &rest[..authority_end];
// return Some(format!("{}://{}", scheme, authority));
// }
// return Some(format!("{}://{}", scheme, rest));
// }
// None
// }
test/rest-api.http
+11 -2
View File
@@ -17,10 +17,19 @@ Content-Type: application/json
{"username": "test", "password": "test"}
### xtream live_categories
GET http://localhost:8901/m3u?username=me&password=me
GET http://localhost:8901/player_api.php?action=get_live_categories&username=xt&password=xt
### xtream live_streams
GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&password=xt&category_id=203
GET http://localhost:8901/player_api.php?action=get_live_streams&username=xt&password=xt&category_id=102
### xtream series_categories
GET http://localhost:8901/player_api.php?action=get_series_categories&username=xt&password=xt
### xtream series
GET http://localhost:8901/player_api.php?action=get_series&username=xt&password=xt&category_id=120
### xtream series info
GET http://localhost:8901/player_api.php?action=get_series_info&username=xt&password=xt&series_id=16441
### xtream series stream
GET http://localhost:8901/series/xt/xt/18130