euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-25 01:02:29 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

Merge pull request #485 from euzu/feature/refactoring_stream_properties

Browse Source 
Breaking Changes

- Data format changed — you must clear your data folder and refresh playlists.
- Batch inputs now require a non-empty name (affects playlist UUIDs).
- New DB viewer options expose on-disk DB content (see CLI).

New Features

- Rich, typed stream metadata and improved info documents for Live/Video/Series.
- New PTT title parser extracting detailed title metadata (year, TMDB, episodes).
- CLI DB viewer: --dbx and --dbm to inspect local DB files.

Improvements

- Numeric channel numbers (tvg-chno), better playlist download/persistence, STRM naming/quality.
- MIME-aware resource caching and more permissive config handling for missing fields.
This commit is contained in:
euzu
2025-12-28 14:53:22 +01:00
committed by GitHub
parent 646cc426ba d29283006f
commit 289aaa4bb2
119 changed files with 11806 additions and 5149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+5
View File
@@ -1,8 +1,10 @@
# Changelog
# 3.3.0 (2025-11-xx)
- !BREAKING CHANGE! Due to some heavy refactoring, the old data format is invalid. You need to clean your data folder and update the playlists
- !BREAKING CHANGE! config.yml threads attribute is now renamed to process_parallel and is a boolean (true or false).
- !BREAKING CHANGE! config.yml adds a reverse proxy config field rewrite_secret to keep resource URLs valid after restart.
- !BREAKING CHANGE! removed `forced_retry_interval_secs`.
- !BREAKING CHANGE! `name` attribute is mandatory for input type batch. The name attribute is used for playlist `uuid` and needs to be stable. The first alias is renamed with the input `name` attribute.
- Avoid blocking the runtime when warming the cache.
- Normalize FileLockManager paths so aliases share the same lock.
- Use async file operations for playlist persistence to avoid blocking the async runtime.
@@ -52,6 +54,9 @@
- Made cache storage more robust. Incomplete downloads will be deleted from cache.
- `kick_secs` added to config.yml `web_ui` config. Default 90 seconds, if a user is kicked from the `web_ui`, they can't connect for this duration.
This setting is also used for sleep-timed streams.
- Added new db-viewer options to print db content.
`tuliprox --dbx /opt/tuliprox/data/all_channels/xtream/video.db`
`tuliprox --dbm /opt/tuliprox/data/all_channels/m3u.db`
# 3.2.0 (2025-11-14)
- Added `name` attribute to Staged Input.
Cargo.lock
Generated
+64 -41
View File
@@ -258,24 +258,19 @@ dependencies = [
]
[[package]]
name = "bincode"
version = "2.0.1"
name = "bit-set"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [
"bincode_derive",
"serde",
"unty",
"bit-vec",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
name = "bit-vec"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bitflags"
@@ -995,6 +990,17 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "fancy-regex"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
dependencies = [
"bit-set",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -1096,7 +1102,7 @@ dependencies = [
[[package]]
name = "frontend"
version = "3.2.24"
version = "3.2.25"
dependencies = [
"anyhow",
"base64",
@@ -1672,7 +1678,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13471584da78061a28306d1359dd0178d8d6fc1c7c80e5e35d27260346e0516a"
dependencies = [
"anymap2",
"bincode 1.3.3",
"bincode",
"gloo-console 0.2.3",
"gloo-utils 0.1.7",
"js-sys",
@@ -1688,7 +1694,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76495d3dd87de51da268fa3a593da118ab43eb7f8809e17eb38d3319b424e400"
dependencies = [
"bincode 1.3.3",
"bincode",
"futures",
"gloo-utils 0.2.0",
"gloo-worker-macros",
@@ -1707,7 +1713,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "085f262d7604911c8150162529cefab3782e91adb20202e8658f7275d2aefe5d"
dependencies = [
"bincode 1.3.3",
"bincode",
"futures",
"gloo-utils 0.2.0",
"gloo-worker-macros",
@@ -1825,6 +1831,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hkdf"
version = "0.12.4"
@@ -2396,6 +2408,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"
dependencies = [
"twox-hash",
]
[[package]]
name = "matchit"
version = "0.8.4"
@@ -3415,6 +3436,25 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rmp"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
dependencies = [
"num-traits",
]
[[package]]
name = "rmp-serde"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"
dependencies = [
"rmp",
"serde",
]
[[package]]
name = "route-recognizer"
version = "0.3.1"
@@ -3558,15 +3598,6 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ruzstd"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ff0cc5e135c8870a775d3320910cd9b564ec036b4dc0b8741629020be63f01"
dependencies = [
"twox-hash",
]
[[package]]
name = "ryu"
version = "1.0.20"
@@ -3793,7 +3824,7 @@ dependencies = [
[[package]]
name = "shared"
version = "3.2.24"
version = "3.2.25"
dependencies = [
"base64",
"bitflags 2.10.0",
@@ -3803,9 +3834,11 @@ dependencies = [
"ciborium",
"enum-iterator",
"fastrand",
"hex",
"indexmap",
"js-sys",
"log",
"lz4_flex",
"path-clean",
"pest",
"pest_derive",
@@ -3813,6 +3846,7 @@ dependencies = [
"serde",
"serde_json",
"url",
"uuid",
"zeroize",
]
@@ -4356,13 +4390,12 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "tuliprox"
version = "3.2.24"
version = "3.2.25"
dependencies = [
"arc-swap",
"async-compression",
"axum",
"base64",
"bincode 2.0.1",
"blake3",
"bytes",
"chrono",
@@ -4374,6 +4407,7 @@ dependencies = [
"deunicode",
"enum-iterator",
"env_logger",
"fancy-regex",
"fastrand",
"filetime",
"flate2",
@@ -4386,6 +4420,7 @@ dependencies = [
"log",
"lol_html",
"mime",
"mime_guess",
"notify",
"openssl",
"paste",
@@ -4397,10 +4432,10 @@ dependencies = [
"rayon",
"regex",
"reqwest",
"rmp-serde",
"rpassword",
"rphonetic",
"rust-argon2",
"ruzstd",
"serde",
"serde_json",
"serde_yaml",
@@ -4481,12 +4516,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "url"
version = "2.5.7"
@@ -4570,12 +4599,6 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "walkdir"
version = "2.5.0"
README.md
+10 -8
View File
@@ -66,6 +66,8 @@ Options:
--healthcheck Healtcheck for docker
--scan-library Scan library directories
--force-library-rescan Force full library rescan
--dbx Database file type: xtream
--dbm Database file type: m3u
```
## 1. `config.yml`
@@ -684,13 +686,6 @@ library:
formats:
- "nfo" # Optionally write Kodi-compatible NFO files
classification:
series_patterns:
- "(?i)s\\d+e\\d+" # S01E02 format
- "(?i)\\d+x\\d+" # 1x02 format
- "(?i)season[\\s._-]*\\d+" # "Season 1" format
- "(?i)episode[\\s._-]*\\d+" # "Episode 1" format
playlist:
movie_category: "Local Movies"
series_category: "Local Series"
@@ -704,6 +699,10 @@ library:
# Force full rescan (ignores modification timestamps)
./tuliprox --force-library-rescan
# Show db content
./tuliprox --dbx /opt/tuliprox/data/all_channels/xtream/video.db
./tuliprox --dbm /opt/tuliprox/data/all_channels/m3u.db
```
**API Endpoints**:
@@ -907,7 +906,7 @@ Input alias definition for same provider with same content but different credent
- sources:
- inputs:
- type: xtream
name: my_provider
name: my_provider # Mandatory: used for playlist UUID generation
url: 'http://provider.net'
username: xyz
password: secret1
@@ -930,6 +929,7 @@ There are 2 batch input types `xtream_batch` and `m3u_batch`.
- sources:
- inputs:
- type: xtream_batch
name: my_provider # Mandatory: used for playlist UUID generation
url: 'file:///home/tuliprox/config/my_provider_batch.csv'
targets:
- name: test
@@ -940,6 +940,8 @@ There are 2 batch input types `xtream_batch` and `m3u_batch`.
my_provider_1;user1;password1;http://my_provider_1.com:80;1;0;2028-11-23 12:34:23
my_provider_2;user2;password2;http://my_provider_2.com:8080;1;0;2028-11-23 12:34:23
```
Important: the first alias is renamed with the `name` from input definition. `my_provider_1` gets `my_provider`.
This is necessary because of playlist uuid generation and assigning same channel numbers on each update.
##### `M3uBatch`
```yaml
backend/Cargo.toml
+4 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "tuliprox"
version = "3.2.24"
version = "3.2.25"
edition = "2021"
rust-version = "1.87.0"
@@ -35,7 +35,7 @@ deunicode = "1.6.2"
mime = "0.3"
log = "0.4"
env_logger = "0.11"
bincode = { version = "2.0", features = ["std", "serde"] }
rmp-serde = "1"
rand = "0.9"
fastrand = "2"
rpassword = "7.4"
@@ -50,7 +50,6 @@ tokio = { version = "1.48", features = ["rt-multi-thread", "parking_lot", "fs"]
#tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
tokio-util = { version = "0.7", features = ["io", "io-util"] }
tempfile = "3.23"
ruzstd = "0.8"
filetime = "0.2"
strsim = "0.11"
rphonetic = "3.0"
@@ -74,6 +73,8 @@ chrono-tz = "0.10.4"
zeroize = "1.8.2"
sysinfo = { version = "0.37.2", default-features = false, features = ["system"] }
uuid = { version = "1", features = ["v4"] }
fancy-regex = "0.17.0"
mime_guess = "2"
# [target.'cfg(not(target_env = "msvc"))'.dependencies]
# tikv-jemallocator = { version = "*", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }
backend/src/api/api_utils.rs
+37 -21
View File
@@ -1,3 +1,4 @@
use crate::BUILD_TIMESTAMP;
use crate::api::endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext};
use crate::api::model::{
create_channel_unavailable_stream, create_custom_video_stream_response,
@@ -13,10 +14,12 @@ use crate::api::panel_api::try_provision_account_on_exhausted;
use crate::model::{ConfigInput, ResourceRetryConfig};
use crate::model::{ConfigTarget, ProxyUserCredentials};
use crate::tools::lru_cache::LRUResourceCache;
use crate::utils::request;
use crate::utils::{request};
use crate::utils::{async_file_reader, async_file_writer, create_new_file_for_write, get_file_extension};
use crate::utils::{debug_if_enabled, trace_if_enabled};
use crate::BUILD_TIMESTAMP;
use crate::auth::Fingerprint;
use crate::utils::request::{content_type_from_ext, parse_range};
use arc_swap::ArcSwapOption;
use axum::http::{header, HeaderMap, HeaderValue, Response, StatusCode};
use axum::response::IntoResponse;
@@ -125,8 +128,6 @@ pub use try_option_bad_request;
pub use try_result_bad_request;
pub use try_result_not_found;
pub use try_unwrap_body;
use crate::auth::Fingerprint;
use crate::utils::request::{content_type_from_ext, parse_range};
pub fn get_server_time() -> String {
chrono::offset::Local::now()
@@ -144,7 +145,7 @@ pub fn get_build_time() -> Option<String> {
}
#[allow(clippy::missing_panics_doc)]
pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoResponse + Send {
pub async fn serve_file(file_path: &Path, mime_type: String) -> impl IntoResponse + Send {
match tokio::fs::try_exists(file_path).await {
Ok(exists) => {
if !exists {
@@ -165,7 +166,7 @@ pub async fn serve_file(file_path: &Path, mime_type: mime::Mime) -> impl IntoRes
try_unwrap_body!(axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
.header(axum::http::header::CONTENT_TYPE, mime_type.to_string())
.header(axum::http::header::CONTENT_TYPE, mime_type)
.header(
axum::http::header::CACHE_CONTROL,
axum::http::header::HeaderValue::from_static("no-cache")
@@ -614,7 +615,7 @@ where
P: PlaylistEntry,
{
let item_type = params.item.get_item_type();
let provider_url = &params.item.get_provider_url();
let provider_url = params.item.get_provider_url();
let redirect_request =
params.user.proxy.is_redirect(item_type) || params.target.is_force_redirect(item_type);
@@ -626,17 +627,17 @@ where
if params.target_type == TargetType::M3u {
if redirect_request || is_dash_request {
let redirect_url = if is_hls_request {
&replace_url_extension(provider_url, HLS_EXT)
Cow::Owned(replace_url_extension(&provider_url, HLS_EXT))
} else {
provider_url
};
let redirect_url = if is_dash_request {
&replace_url_extension(redirect_url, DASH_EXT)
Cow::Owned(replace_url_extension(&redirect_url, DASH_EXT))
} else {
redirect_url
};
let redirect_url =
get_redirect_alternative_url(app_state, redirect_url, params.input).await;
get_redirect_alternative_url(app_state, &redirect_url, params.input).await;
debug_if_enabled!(
"Redirecting stream request to {}",
sanitize_sensitive_info(&redirect_url)
@@ -669,8 +670,8 @@ where
let stream_url = match get_xtream_player_api_stream_url(
params.input,
params.req_context,
&params.get_query_path(provider_id, provider_url),
provider_url,
&params.get_query_path(provider_id, &provider_url),
&provider_url,
) {
None => {
error!("Cant find stream url for target {target_name}, context {}, stream_id {virtual_id}", params.req_context);
@@ -1208,25 +1209,39 @@ pub fn get_headers_from_request(
fn get_add_cache_content(
res_url: &str,
mime_type: Option<String>,
cache: &Arc<ArcSwapOption<Mutex<LRUResourceCache>>>,
) -> Arc<dyn Fn(usize) + Send + Sync> {
let resource_url = String::from(res_url);
let cache = Arc::clone(cache);
let add_cache_content: Arc<dyn Fn(usize) + Send + Sync> = Arc::new(move |size| {
let res_url = resource_url.clone();
let mime_type = mime_type.clone();
// todo spawn, replace with unboundchannel
let cache = Arc::clone(&cache);
tokio::spawn(async move {
if let Some(cache) = cache.load().as_ref() {
let _ = cache.lock().await.add_content(&res_url, size);
let _ = cache.lock().await.add_content(&res_url, mime_type, size);
}
});
});
add_cache_content
}
async fn build_stream_response(
fn get_mime_type(headers: &axum::http::HeaderMap, resource_url: &str) -> Option<String> {
headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok()) // Option<&str>
.map(ToString::to_string) // Option<String>
.or_else(|| {
// fallback to guess
mime_guess::from_path(resource_url)
.first_raw()
.map(ToString::to_string)
})
}
async fn build_resource_stream_response(
app_state: &Arc<AppState>,
resource_url: &str,
response: reqwest::Response,
@@ -1235,6 +1250,7 @@ async fn build_stream_response(
let status = response.status();
let mut response_builder =
axum::response::Response::builder().status(status);
let mime_type = get_mime_type(response.headers(), resource_url);
let has_content_range = response.headers().contains_key(axum::http::header::CONTENT_RANGE);
for (key, value) in response.headers() {
let name = key.as_str();
@@ -1261,7 +1277,7 @@ async fn build_stream_response(
if can_cache {
debug!( "Caching eligible resource stream {sanitized_resource_url}");
let cache_resource_path = if let Some(cache) = app_state.cache.load().as_ref() {
Some(cache.lock().await.store_path(resource_url))
Some(cache.lock().await.store_path(resource_url, mime_type.as_deref()))
} else {
None
};
@@ -1270,7 +1286,7 @@ async fn build_stream_response(
Ok(file) => {
debug!("Persisting resource stream {sanitized_resource_url} to {}", resource_path.display());
let writer = async_file_writer(file);
let add_cache_content = get_add_cache_content(resource_url, &app_state.cache);
let add_cache_content = get_add_cache_content(resource_url, mime_type, &app_state.cache);
let tee = tee_stream(byte_stream, writer, &resource_path, add_cache_content);
return try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(tee)));
}
@@ -1313,7 +1329,7 @@ async fn fetch_resource_with_retry(
let status = response.status();
if status.is_success() {
return Some(
build_stream_response(app_state, resource_url, response).await,
build_resource_stream_response(app_state, resource_url, response).await,
);
}
// Retry only for 408, 425, 429 and all 5xx statuses
@@ -1405,9 +1421,9 @@ pub async fn resource_response(
let req_headers = get_headers_from_request(req_headers, &filter);
if let Some(cache) = app_state.cache.load().as_ref() {
let mut guard = cache.lock().await;
if let Some(resource_path) = guard.get_content(resource_url) {
if let Some((resource_path, mime_type)) = guard.get_content(resource_url) {
trace_if_enabled!("Responding resource from cache {}", sanitize_sensitive_info(resource_url));
return serve_file(&resource_path, mime::APPLICATION_OCTET_STREAM)
return serve_file(&resource_path, mime_type.unwrap_or_else(|| mime::APPLICATION_OCTET_STREAM.to_string()))
.await
.into_response();
}
@@ -1513,7 +1529,7 @@ pub fn json_response<T: Serialize>(data: &T) -> impl IntoResponse + Send {
(axum::http::StatusCode::OK, axum::Json(data)).into_response()
}
pub fn json_or_bin_response<T: Serialize>(accept: Option<&String>, data: &T) -> impl IntoResponse + Send {
pub fn json_or_bin_response<T: Serialize>(accept: Option<&str>, data: &T) -> impl IntoResponse + Send {
if accept.is_some_and(|a| a.contains(CONTENT_TYPE_BIN)) {
return bin_response(data).into_response();
}
backend/src/api/endpoints/api_playlist_utils.rs
+5 -5
View File
@@ -85,7 +85,7 @@ fn group_playlist_groups_by_cluster(playlist: Vec<PlaylistGroup>) -> (Vec<Playli
let mut series = Vec::new();
for group in playlist {
let channels = group.channels.iter()
.map(shared::model::PlaylistItem::to_common)
.map(CommonPlaylistItem::from)
.collect();
let grp = PlaylistResponseGroup {
id: group.id,
@@ -115,7 +115,7 @@ async fn grouped_channels(
))
}
pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&String>) -> impl axum::response::IntoResponse + Send {
pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Option<&ConfigTarget>, cfg: &AppConfig, accept: Option<&str>) -> impl axum::response::IntoResponse + Send {
if let Some(target) = cfg_target {
if target.has_output(TargetType::Xtream) {
let live_channels = grouped_channels(cfg, target, XtreamCluster::Live).await;
@@ -144,13 +144,13 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target(cfg_target: Optio
(axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid Arguments"}))).into_response()
}
pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cfg_input: Option<&Arc<ConfigInput>>, cfg: &Arc<Config>, accept: Option<&String>) -> impl IntoResponse + Send {
pub(in crate::api::endpoints) async fn get_playlist(client: &reqwest::Client, cfg_input: Option<&Arc<ConfigInput>>, cfg: &Arc<Config>, accept: Option<&str>) -> impl IntoResponse + Send {
match cfg_input {
Some(input) => {
let (result, errors) =
match input.input_type {
InputType::M3u | InputType::M3uBatch => m3u::get_m3u_playlist(client, cfg, input).await,
InputType::Xtream | InputType::XtreamBatch => xtream::get_xtream_playlist(cfg, client, input).await,
InputType::M3u | InputType::M3uBatch => m3u::download_m3u_playlist(client, cfg, input).await,
InputType::Xtream | InputType::XtreamBatch => xtream::download_xtream_playlist(cfg, client, input).await,
InputType::Library => {
return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({ "error": "Library inputs are not supported on this endpoint"}))).into_response();
}
backend/src/api/endpoints/hdhomerun_api.rs
+3 -5
View File
@@ -15,7 +15,7 @@ use serde_json::json;
use shared::model::{
M3uPlaylistItem, PlaylistItemType, TargetType, XtreamCluster, XtreamPlaylistItem,
};
use shared::utils::{concat_path, get_string_from_serde_value};
use shared::utils::{concat_path};
use std::sync::Arc;
#[derive(Serialize, Deserialize, Clone)]
@@ -112,9 +112,7 @@ where
o.xtream_live_stream_without_extension,
)
});
let container_extension = item
.get_additional_property("container_extension")
.map(|v| get_string_from_serde_value(&v).unwrap_or_default());
let container_extension = item.get_container_extension();
let stream_url = match &base_url {
None => item.url.clone(),
Some(url) => get_xtream_url(
@@ -123,7 +121,7 @@ where
&credentials.username,
&credentials.password,
item.virtual_id,
container_extension.as_ref(),
container_extension.as_deref(),
live_stream_use_prefix,
live_stream_without_extension,
),
backend/src/api/endpoints/hls_api.rs
+1 -1
View File
@@ -174,7 +174,7 @@ pub(in crate::api) async fn handle_hls_stream_request(
async fn get_stream_channel(app_state: &Arc<AppState>, target: &Arc<ConfigTarget>, virtual_id: u32) -> Option<StreamChannel> {
if target.has_output(TargetType::Xtream) {
if let Ok((pli, _)) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await {
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await {
return Some(pli.to_stream_channel(target.id));
}
}
backend/src/api/endpoints/v1_api.rs
+3 -2
View File
@@ -13,7 +13,7 @@ use shared::utils::{concat_path_leading_slash};
use std::collections::{BTreeMap, HashMap};
use std::io::{Cursor};
use std::sync::Arc;
use log::error;
use log::{error, info};
use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader;
use crate::api::endpoints::v1_api_config::v1_api_config_register;
use crate::api::endpoints::library_api::library_api_register;
@@ -71,7 +71,7 @@ async fn status(axum::extract::State(app_state): axum::extract::State<Arc<AppSta
async fn streams(ExtractAcceptHeader(accept): ExtractAcceptHeader,
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>) -> axum::response::Response {
let streams = app_state.active_users.active_streams().await;
json_or_bin_response(accept.as_ref(), &streams).into_response()
json_or_bin_response(accept.as_deref(), &streams).into_response()
}
async fn geoip_update(axum::extract::State(app_state): axum::extract::State<Arc<AppState>>) -> axum::response::Response {
@@ -106,6 +106,7 @@ async fn geoip_update(axum::extract::State(app_state): axum::extract::State<Arc<
return match result {
(Some(_), None) => {
info!("GeoIp db updated");
app_state.geoip.store(Some(Arc::new(geoip)));
axum::http::StatusCode::OK.into_response()
},
backend/src/api/endpoints/v1_api_playlist.rs
+6 -7
View File
@@ -80,7 +80,6 @@ async fn playlist_update(
}
}
async fn playlist_content(
ExtractAcceptHeader(accept): ExtractAcceptHeader,
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>,
@@ -90,16 +89,16 @@ async fn playlist_content(
let client = app_state.http_client.load();
match playlist_req {
PlaylistRequest::Target(target_id) => {
get_playlist_for_target(app_state.app_config.get_target_by_id(target_id).as_deref(), &app_state.app_config, accept.as_ref()).await.into_response()
get_playlist_for_target(app_state.app_config.get_target_by_id(target_id).as_deref(), &app_state.app_config, accept.as_deref()).await.into_response()
}
PlaylistRequest::Input(input_id) => {
get_playlist(client.as_ref(), app_state.app_config.get_input_by_id(input_id).as_ref(), &config, accept.as_ref()).await.into_response()
get_playlist(client.as_ref(), app_state.app_config.get_input_by_id(input_id).as_ref(), &config, accept.as_deref()).await.into_response()
}
PlaylistRequest::CustomXtream(xtream) => {
match Url::parse(&xtream.url) {
Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => {
let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url));
get_playlist(client.as_ref(), Some(&input), &config, accept.as_ref()).await.into_response()
get_playlist(client.as_ref(), Some(&input), &config, accept.as_deref()).await.into_response()
}
_ => {
(axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response()
@@ -110,7 +109,7 @@ async fn playlist_content(
match Url::parse(&m3u.url) {
Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => {
let input = Arc::new(create_config_input_for_m3u(&m3u.url));
get_playlist(client.as_ref(), Some(&input), &config, accept.as_ref()).await.into_response()
get_playlist(client.as_ref(), Some(&input), &config, accept.as_deref()).await.into_response()
}
_ => {
(axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"}))).into_response()
@@ -143,7 +142,7 @@ async fn playlist_epg(
let config = &app_state.app_config.config.load();
if let Some(epg_path) = crate::api::endpoints::xmltv_api::get_epg_path_for_target(config, &target) {
if let Ok(epg) = parse_xmltv_for_web_ui_from_file(&epg_path).await {
return json_or_bin_response(accept.as_ref(), &epg).into_response();
return json_or_bin_response(accept.as_deref(), &epg).into_response();
}
}
}
@@ -161,7 +160,7 @@ async fn playlist_epg(
}
PlaylistEpgRequest::Custom(url) => {
if let Ok(epg) = parse_xmltv_for_web_ui_from_url(&app_state, &url).await {
return json_or_bin_response(accept.as_ref(), &epg).into_response();
return json_or_bin_response(accept.as_deref(), &epg).into_response();
}
}
}
backend/src/api/endpoints/web_index.rs
+2 -2
View File
@@ -219,7 +219,7 @@ async fn index(
error!("Failed to read web ui index.html: {err}");
}
}
serve_file(&path, mime::TEXT_HTML_UTF_8).await.into_response()
serve_file(&path, mime::TEXT_HTML_UTF_8.to_string()).await.into_response()
}
async fn index_config(
@@ -279,7 +279,7 @@ async fn index_config(
}
}
}
serve_file(&path, mime::APPLICATION_JSON).await.into_response()
serve_file(&path, mime::APPLICATION_JSON.to_string()).await.into_response()
}
pub fn index_register_without_path(web_dir_path: &Path) -> axum::Router<Arc<AppState>> {
backend/src/api/endpoints/websocket_api.rs
+2 -2
View File
@@ -305,7 +305,7 @@ async fn handle_socket(mut socket: WebSocket, app_state: Arc<AppState>, auth_req
maybe_msg = socket.recv() => {
if let Some(msg) = maybe_msg {
if let Err(e) = handle_incoming_message(msg, &mut socket, &mut handler, &app_state, auth_required, secret_key.as_ref()).await {
error!("WebSocket message handling error: {e}");
trace!("WebSocket message handling error: {e}");
break;
}
} else {
@@ -315,7 +315,7 @@ async fn handle_socket(mut socket: WebSocket, app_state: Arc<AppState>, auth_req
Ok(event) = event_rx.recv() => {
if let Err(e) = handle_event_message(&mut socket, event, &handler).await {
error!("Failed to send ws event: {e}");
trace!("Failed to send ws event: {e}");
break;
}
}
backend/src/api/endpoints/xmltv_api.rs
+3 -3
View File
@@ -121,7 +121,7 @@ pub(in crate::api) fn get_epg_path_for_target(config: &Config, target: &ConfigTa
/// or a timezone name (e.g. "`Europe/Berlin`", "`UTC`", "`America/New_York`").
///
/// Returns the total offset in minutes (i32).
fn parse_timeshift(time_shift: Option<&String>) -> Option<i32> {
fn parse_timeshift(time_shift: Option<&str>) -> Option<i32> {
time_shift.and_then(|offset| {
// Try to parse as timezone name first
if let Ok(tz) = offset.parse::<Tz>() {
@@ -164,7 +164,7 @@ pub async fn serve_epg(
let rewrite_urls = !redirect && rewrite_resources;
// Use 0 for timeshift if None
let timeshift = parse_timeshift(user.epg_timeshift.as_ref()).unwrap_or(0);
let timeshift = parse_timeshift(user.epg_timeshift.as_deref()).unwrap_or(0);
return if timeshift != 0 || rewrite_urls || filter.is_some() {
let server_info = app_state.app_config.get_user_server_info(user);
@@ -174,7 +174,7 @@ pub async fn serve_epg(
serve_epg_with_rewrites(epg_path, timeshift, rewrite_urls, &encrypt_secret, &base_url, filter).await
} else {
// Neither timeshift nor rewrite needed, serve original file
serve_file(epg_path, mime::TEXT_XML).await.into_response()
serve_file(epg_path, mime::TEXT_XML.to_string()).await.into_response()
};
}
}
backend/src/api/endpoints/xtream_api.rs
+46 -223
View File
@@ -1,5 +1,4 @@
// https://github.com/tellytv/go.xtream-codes/blob/master/structs.go
use crate::api::api_utils;
use crate::api::api_utils::{create_session_fingerprint, local_stream_response, try_unwrap_body};
use crate::api::api_utils::{
@@ -15,7 +14,7 @@ use crate::api::model::UserApiRequest;
use crate::api::model::XtreamAuthorizationResponse;
use crate::api::model::{create_custom_video_stream_response, CustomVideoStreamType};
use crate::auth::Fingerprint;
use crate::model::{AppConfig, ConfigTarget};
use crate::model::{xtream_mapping_option_from_target_options, ConfigTarget};
use crate::model::{Config, ConfigInput};
use crate::model::{InputSource, ProxyUserCredentials};
use crate::repository::playlist_repository::get_target_id_mapping;
@@ -33,15 +32,15 @@ use futures::Stream;
use log::{debug, error, warn};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use shared::error::create_tuliprox_error_result;
use shared::error::info_err;
use shared::error::create_tuliprox_error_result;
use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind};
use shared::model::{create_stream_channel_with_type, get_backdrop_path_value, FieldGetAccessor, PlaylistEntry, PlaylistItemType, ProxyType, TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem};
use shared::model::{create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType,
TargetType, UserConnectionPermission, XtreamCluster};
use shared::utils::{
extract_extension_from_url, generate_playlist_uuid, get_u32_from_serde_value, hex_encode,
sanitize_sensitive_info, trim_slash, HLS_EXT,
extract_extension_from_url, generate_playlist_uuid, sanitize_sensitive_info,
trim_slash, HLS_EXT,
};
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::fmt::{Display, Formatter};
use std::path::Path;
@@ -67,12 +66,12 @@ impl ApiStreamContext {
impl Display for ApiStreamContext {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}",
match self {
Self::Live | Self::LiveAlt => Self::LIVE,
Self::Movie => Self::MOVIE,
Self::Series => Self::SERIES,
Self::Timeshift => Self::TIMESHIFT,
}
match self {
Self::Live | Self::LiveAlt => Self::LIVE,
Self::Movie => Self::MOVIE,
Self::Series => Self::SERIES,
Self::Timeshift => Self::TIMESHIFT,
}
)
}
}
@@ -247,7 +246,7 @@ async fn xtream_player_api_stream(
let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id);
let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse());
let (pli, mapping) = try_result_not_found!(
let pli = try_result_not_found!(
xtream_repository::xtream_get_item_for_stream_id(req_virtual_id, app_state, &target, None).await,
true,
format!("Failed to read xtream item for stream id {req_virtual_id}")
@@ -347,7 +346,7 @@ async fn xtream_player_api_stream(
let redirect_params = RedirectParams {
item: &pli,
provider_id: Some(mapping.provider_id),
provider_id: pli.get_provider_id(),
cluster,
target_type: TargetType::Xtream,
target: &target,
@@ -436,7 +435,7 @@ async fn xtream_player_api_stream_with_token(
}
let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id);
let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse());
let (pli, _mapping) = try_result_bad_request!(
let pli = try_result_bad_request!(
xtream_repository::xtream_get_item_for_stream_id(
req_virtual_id,
app_state,
@@ -565,163 +564,6 @@ async fn xtream_player_api_stream_with_token(
}
}
fn get_doc_id_and_field_name(input: &str) -> Option<(u32, &str)> {
if let Some(pos) = input.find('_') {
let (number_part, rest) = input.split_at(pos);
let field = &rest[1..]; // cut _
if let Ok(number) = number_part.parse::<u32>() {
return Some((number, field));
}
}
None
}
fn get_doc_resource_field_value<'a>(
field: &'a str,
doc: Option<&'a Value>,
) -> Option<Cow<'a, str>> {
if let Some(Value::Object(info_data)) = doc {
if field.starts_with(crate::model::XC_PROP_BACKDROP_PATH) {
return get_backdrop_path_value(
field,
info_data.get(crate::model::XC_PROP_BACKDROP_PATH),
);
} else if let Some(Value::String(url)) = info_data.get(field) {
return Some(Cow::Borrowed(url));
}
}
None
}
async fn xtream_get_info_resource_url<'a>(
config: &'a AppConfig,
pli: &'a XtreamPlaylistItem,
target: &'a ConfigTarget,
resource: &'a str,
) -> Result<Option<Cow<'a, str>>, serde_json::Error> {
let info_content = match pli.xtream_cluster {
XtreamCluster::Video => xtream_repository::xtream_load_vod_info(
config,
target.name.as_str(),
pli.get_virtual_id(),
).await,
XtreamCluster::Series => xtream_repository::xtream_load_series_info(
config,
target.name.as_str(),
pli.get_virtual_id(),
).await,
XtreamCluster::Live => None,
};
if let Some(content) = info_content {
let doc: Map<String, Value> = serde_json::from_str(&content)?;
let (field, possible_episode_id) = if let Some(field_name_with_episode_id) =
resource.strip_prefix(crate::model::XC_INFO_RESOURCE_PREFIX_EPISODE)
{
if let Some((episode_id, field_name)) =
get_doc_id_and_field_name(field_name_with_episode_id)
{
(field_name, Some(episode_id))
} else {
return Ok(None);
}
} else {
(
&resource[crate::model::XC_INFO_RESOURCE_PREFIX.len()..],
None,
)
};
let info_doc = match pli.xtream_cluster {
XtreamCluster::Video | XtreamCluster::Series => {
if let Some(episode_id) = possible_episode_id {
get_episode_info_doc(&doc, episode_id)
} else {
doc.get(crate::model::XC_TAG_INFO_DATA)
}
}
XtreamCluster::Live => None,
};
if let Some(value) = get_doc_resource_field_value(field, info_doc) {
return Ok(Some(Cow::Owned(value.into_owned())));
}
}
Ok(None)
}
fn get_episode_info_doc(doc: &Map<String, Value>, episode_id: u32) -> Option<&Value> {
let episodes = doc.get(crate::model::XC_TAG_EPISODES)?.as_object()?;
for season_episodes in episodes.values() {
if let Value::Array(episode_list) = season_episodes {
for episode in episode_list {
if let Value::Object(episode_doc) = episode {
if let Some(episode_id_value) = episode_doc.get(crate::model::XC_TAG_ID) {
if let Some(doc_episode_id) = get_u32_from_serde_value(episode_id_value) {
if doc_episode_id == episode_id {
return episode_doc.get(crate::model::XC_TAG_INFO_DATA);
}
}
}
}
}
}
}
None
}
fn get_season_info_doc(doc: &Vec<Value>, season_id: u32) -> Option<&Value> {
for season in doc {
if let Value::Object(season_doc) = season {
if let Some(season_id_value) = season_doc.get(crate::model::XC_TAG_ID) {
if let Some(doc_season_id) = get_u32_from_serde_value(season_id_value) {
if doc_season_id == season_id {
return Some(season);
}
}
}
}
}
None
}
async fn xtream_get_season_resource_url<'a>(
config: &'a AppConfig,
pli: &'a XtreamPlaylistItem,
target: &'a ConfigTarget,
resource: &'a str,
) -> Result<Option<Cow<'a, str>>, serde_json::Error> {
let info_content = match pli.xtream_cluster {
XtreamCluster::Series => xtream_repository::xtream_load_series_info(
config,
target.name.as_str(),
pli.get_virtual_id(),
).await,
XtreamCluster::Video | XtreamCluster::Live => None,
};
if let Some(content) = info_content {
let doc: Map<String, Value> = serde_json::from_str(&content)?;
if let Some(field_name_with_season_id) =
resource.strip_prefix(crate::model::XC_SEASON_RESOURCE_PREFIX)
{
if let Some((season_id, field)) = get_doc_id_and_field_name(field_name_with_season_id) {
let seasons_doc = match pli.xtream_cluster {
XtreamCluster::Series => doc.get(crate::model::XC_TAG_SEASONS_DATA),
XtreamCluster::Video | XtreamCluster::Live => None,
};
if let Some(Value::Array(seasons)) = seasons_doc {
if let Some(value) =
get_doc_resource_field_value(field, get_season_info_doc(seasons, season_id))
{
return Ok(Some(Cow::Owned(value.into_owned())));
}
}
}
}
}
Ok(None)
}
async fn xtream_player_api_resource(
req_headers: &HeaderMap,
api_req: &UserApiRequest,
@@ -751,7 +593,7 @@ async fn xtream_player_api_resource(
}
let req_virtual_id: u32 = try_result_bad_request!(resource_req.stream_id.trim().parse());
let resource = resource_req.action_path.trim();
let (pli, _) = try_result_bad_request!(
let pli = try_result_bad_request!(
xtream_repository::xtream_get_item_for_stream_id(
req_virtual_id,
app_state,
@@ -761,23 +603,8 @@ async fn xtream_player_api_resource(
true,
format!("Failed to read xtream item for stream id {req_virtual_id}")
);
let stream_url = if resource.starts_with(crate::model::XC_INFO_RESOURCE_PREFIX) {
try_result_bad_request!(xtream_get_info_resource_url(
&app_state.app_config,
&pli,
&target,
resource
).await)
} else if resource.starts_with(crate::model::XC_SEASON_RESOURCE_PREFIX) {
try_result_bad_request!(xtream_get_season_resource_url(
&app_state.app_config,
&pli,
&target,
resource
).await)
} else {
pli.get_field(resource)
};
let stream_url = pli.resolve_resource_url(resource);
match stream_url {
None => axum::http::StatusCode::NOT_FOUND.into_response(),
@@ -790,9 +617,7 @@ async fn xtream_player_api_resource(
redirect(&url).into_response()
} else {
trace_if_enabled!("Resource request to {}", sanitize_sensitive_info(&url));
resource_response(app_state, &url, req_headers, None)
.await
.into_response()
resource_response(app_state, &url, req_headers, None).await.into_response()
}
}
}
@@ -977,6 +802,16 @@ async fn xtream_player_api_timeshift_query_stream(
.into_response()
}
fn empty_json_response() -> axum::http::Result<axum::response::Response> {
axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
.header(
axum::http::header::CONTENT_TYPE,
mime::APPLICATION_JSON.to_string(),
)
.body(axum::body::Body::from("{}".as_bytes()))
}
async fn xtream_get_stream_info_response(
app_state: &Arc<AppState>,
user: &ProxyUserCredentials,
@@ -989,31 +824,26 @@ async fn xtream_get_stream_info_response(
Err(_) => return axum::http::StatusCode::BAD_REQUEST.into_response(),
};
if let Ok((pli, virtual_record)) = xtream_repository::xtream_get_item_for_stream_id(
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(
virtual_id,
app_state,
target,
Some(cluster),
).await {
if pli.item_type.is_local() {
if let Some(additional_properties) = pli.additional_properties.as_ref() {
let bytes = additional_properties.as_bytes().to_vec();
return try_unwrap_body!(axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
.header(
axum::http::header::CONTENT_TYPE,
mime::APPLICATION_JSON.as_ref()
)
.body(axum::body::Body::from(bytes)));
}
let Ok(xtream_output) = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name))) else {
return try_unwrap_body!(empty_json_response());
};
let server_info = app_state.app_config.get_user_server_info(user);
let options = xtream_mapping_option_from_target_options(target, xtream_output, &app_state.app_config, user, Some(server_info.get_base_url().as_str()));
return axum::Json(pli.to_info_document(&options)).into_response();
}
if pli.provider_id > 0 {
let input_name = &pli.input_name;
if let Some(input) = app_state.app_config.get_input_by_name(input_name.as_str()) {
if let Some(info_url) =
xtream::get_xtream_player_api_info_url(&input, cluster, pli.provider_id)
{
if let Some(info_url) = xtream::get_xtream_player_api_info_url(&input, cluster, pli.provider_id) {
// Redirect is only possible for live streams, vod and series info needs to be modified
if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live {
return redirect(&info_url).into_response();
@@ -1026,8 +856,7 @@ async fn xtream_get_stream_info_response(
&pli,
info_url.as_str(),
cluster,
)
.await
).await
{
return try_unwrap_body!(axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
@@ -1044,7 +873,7 @@ async fn xtream_get_stream_info_response(
return match cluster {
XtreamCluster::Video => {
let content =
create_vod_info_from_item(target, user, &pli, virtual_record.last_updated);
create_vod_info_from_item(target, user, &pli);
try_unwrap_body!(axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
.header(
@@ -1064,13 +893,7 @@ async fn xtream_get_stream_info_response(
}
};
}
try_unwrap_body!(axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
.header(
axum::http::header::CONTENT_TYPE,
mime::APPLICATION_JSON.to_string()
)
.body(axum::body::Body::from("{}".as_bytes())))
try_unwrap_body!(empty_json_response())
}
async fn xtream_get_short_epg(
@@ -1087,7 +910,7 @@ async fn xtream_get_short_epg(
Err(_) => return axum::http::StatusCode::BAD_REQUEST.into_response(),
};
if let Ok((pli, _)) = xtream_repository::xtream_get_item_for_stream_id(
if let Ok(pli) = xtream_repository::xtream_get_item_for_stream_id(
virtual_id,
app_state,
target,
@@ -1236,7 +1059,7 @@ async fn xtream_player_api_handle_content_action(
);
}
return Some(
serve_file(&file_path, mime::APPLICATION_JSON)
serve_file(&file_path, mime::APPLICATION_JSON.to_string())
.await
.into_response(),
);
@@ -1258,7 +1081,7 @@ async fn xtream_get_catchup_response(
end: &str,
) -> impl IntoResponse + Send {
let req_virtual_id: u32 = try_result_bad_request!(FromStr::from_str(stream_id));
let (pli, _) = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(
let pli = try_result_bad_request!(xtream_repository::xtream_get_item_for_stream_id(
req_virtual_id,
app_state,
target,
@@ -1302,10 +1125,10 @@ async fn xtream_get_catchup_response(
.and_then(|id| id.parse::<u32>().ok())
{
let uuid = generate_playlist_uuid(
&hex_encode(&pli.get_uuid()),
&pli.get_uuid().to_string(),
&catchup_provider_id.to_string(),
pli.item_type,
&pli.url,
&pli.input_name,
);
let virtual_id = target_id_mapping.get_and_update_virtual_id(
&uuid,
backend/src/api/main_api.rs
+1 -1
View File
@@ -81,7 +81,7 @@ async fn create_shared_data(
Arc::new(ArcSwapOption::from(Some(Arc::new(db))))
}
Err(err) => {
error!("Failed to load GeoIp db: {err}");
info!("No GeoIp db found: {err}");
Arc::new(ArcSwapOption::from(None))
}
}
backend/src/api/model/playlist_mem_cache.rs
+33 -1
View File
@@ -1,6 +1,6 @@
use std::collections::HashMap;
use tokio::sync::RwLock;
use shared::model::{M3uPlaylistItem, XtreamPlaylistItem};
use shared::model::{M3uPlaylistItem, PlaylistItem, XtreamCluster, XtreamPlaylistItem};
use crate::model::ConfigTarget;
use crate::repository::bplustree::BPlusTree;
use crate::repository::target_id_mapping::{VirtualIdRecord};
@@ -50,6 +50,38 @@ impl PlaylistStorageState {
}
}
pub async fn update_playlist_items(&self, target: &ConfigTarget, pli_list: Vec<&XtreamPlaylistItem>) {
if target.use_memory_cache {
if let Some(storage) = self.data.write().await.get_mut(&target.name) {
if let Some(xtream) = storage.xtream.as_mut() {
for pli in pli_list {
match pli.xtream_cluster {
XtreamCluster::Live => &mut xtream.live,
XtreamCluster::Video => &mut xtream.vod,
XtreamCluster::Series => &mut xtream.series,
}.insert(pli.virtual_id, pli.clone());
}
}
}
}
}
pub async fn insert_playlist_items(&self, target: &ConfigTarget, pli_list: Vec<PlaylistItem>) {
if target.use_memory_cache {
if let Some(storage) = self.data.write().await.get_mut(&target.name) {
if let Some(xtream) = storage.xtream.as_mut() {
for pli in pli_list {
match pli.header.xtream_cluster {
XtreamCluster::Live => &mut xtream.live,
XtreamCluster::Video => &mut xtream.vod,
XtreamCluster::Series => &mut xtream.series,
}.insert(pli.header.virtual_id, XtreamPlaylistItem::from(&pli));
}
}
}
}
}
pub async fn cache_playlist(&self, target_name: &str, playlist: PlaylistStorage) {
match playlist {
PlaylistStorage::M3uPlaylist(m3u_playlist) => {
backend/src/library/classifier.rs
+58 -265
View File
@@ -1,68 +1,24 @@
use log::debug;
use regex::Regex;
use std::path::Path;
use shared::utils::CONSTANTS;
use crate::model::LibraryConfig;
use crate::library::scanner::ScannedMediaFile;
use crate::library::{SeriesEpisodeFile, SeriesKey};
use crate::library::{SeriesKey};
use crate::ptt::{ptt_parse_title, PttMetadata};
fn clear_filename(file_name: &str) -> String {
// Remove quality indicators (1080p, 720p, BluRay, etc.)
let mut cleaned = CONSTANTS.re_quality.replace(file_name, "").to_string();
// Clean up special characters
cleaned = cleaned.replace(['.', '_', '-'], " ").trim().to_string();
// remove brackets
cleaned = CONSTANTS.re_classifier_brackets_info.replace_all(&cleaned, "").to_string();
// put space between camel case
cleaned = CONSTANTS.re_classifier_camel_case.replace_all(&cleaned, "$1 $2").to_string();
// Remove multiple spaces
CONSTANTS.re_whitespace.replace_all(&cleaned, " ").trim().to_string()
}
#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq)]
pub enum MovieDbId {
Tmdb(u32),
Tvdb(u32),
}
impl MovieDbId {
fn get_id<F>(ids: Option<&Vec<MovieDbId>>, f: F) -> Option<u32>
where
F: Fn(&MovieDbId) -> Option<u32>,
{
ids.as_ref()?.iter().find_map(f)
}
pub fn get_tmdb_id(ids: Option<&Vec<MovieDbId>>) -> Option<u32> {
Self::get_id(ids, |id| {
if let MovieDbId::Tmdb(val) = id { Some(*val) } else { None }
})
}
pub fn get_tvdb_id(ids: Option<&Vec<MovieDbId>>) -> Option<u32> {
Self::get_id(ids, |id| {
if let MovieDbId::Tvdb(val) = id { Some(*val) } else { None }
})
}
}
/// Classification result for a video file
// Classification result for a video file
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MediaClassification {
Movie,
Movie {
metadata: PttMetadata
},
Series {
season: Option<u32>,
episode: Option<u32>,
key: SeriesKey,
episode: u32,
season: u32,
metadata: PttMetadata,
},
}
impl MediaClassification {
pub fn is_movie(&self) -> bool {
matches!(self, MediaClassification::Movie)
matches!(self, MediaClassification::Movie { .. })
}
pub fn is_series(&self) -> bool {
@@ -72,171 +28,29 @@ impl MediaClassification {
/// Classifier for determining if a video file is a movie or series
pub struct MediaClassifier {
series_patterns: Vec<Regex>,
}
impl MediaClassifier {
/// Creates a new classifier from the VOD configuration
pub fn from_config(config: &LibraryConfig) -> Self {
Self {
series_patterns: config.classification.series_patterns.clone(),
}
}
// Classifies a video file as either Movie or Series
pub fn classify(&self, file: &ScannedMediaFile) -> MediaClassification {
pub fn classify(file: &ScannedMediaFile) -> MediaClassification {
let file_name = &file.file_name;
let parent_path = file.path.parent().and_then(|p| p.to_str()).unwrap_or("");
// Check if any series pattern matches
for pattern in &self.series_patterns {
if pattern.is_match(file_name) || pattern.is_match(parent_path) {
debug!("File '{file_name}' matched series pattern: {pattern}");
return Self::extract_series_info(file_name, parent_path);
let ptt_metadata = ptt_parse_title(file_name);
if let (Some(episode), Some(season)) = (ptt_metadata.episodes.first(), ptt_metadata.seasons.first()) {
MediaClassification::Series {
key: SeriesKey {
title: ptt_metadata.title.clone(),
year: ptt_metadata.year,
tmdb_id: ptt_metadata.tmdb,
},
episode: *episode,
season: *season,
metadata: ptt_metadata,
}
} else {
MediaClassification::Movie {metadata: ptt_metadata }
}
// If no series pattern matches, classify as movie
debug!("File '{file_name}' classified as Movie (no series pattern match)");
MediaClassification::Movie
}
pub fn classify_series_file(file: &ScannedMediaFile) -> Option<(SeriesKey, u32, u32)> {
let file_name = Path::new(&file.file_name)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(&file.file_name);
let (ids, show_name) = Self::extract_show_name(file);
let parent_path = file.path.parent()
.and_then(|p| p.to_str())
.unwrap_or("");
let classification = Self::extract_series_info(file_name, parent_path);
if let MediaClassification::Series { season, episode } = classification {
if let (Some(season), Some(episode)) = (season, episode) {
let tmdb_id = MovieDbId::get_tmdb_id(ids.as_ref());
let year = None; // TODO extract from somewhere
let key = SeriesKey {
title: show_name,
tmdb_id,
year,
};
return Some((key, season, episode));
}
}
None
}
/// Extracts season and episode information from file name or path
fn extract_series_info(file_name: &str, parent_path: &str) -> MediaClassification {
let combined = format!("{parent_path} {file_name}");
// Try to extract season and episode numbers
// Common patterns: S01E02, s01e02, 1x02, Season 1 Episode 2, etc.
let season = Self::extract_season(&combined);
let episode = Self::extract_episode(&combined);
if season.is_some() || episode.is_some() {
debug!("Extracted series info - Season: {season:?}, Episode: {episode:?}");
}
MediaClassification::Series { season, episode }
}
/// Extracts season number from text using common patterns
fn extract_season(text: &str) -> Option<u32> {
// Patterns: S01, s01, Season 1, season 1, Season1, etc.
CONSTANTS.re_classifier_season
.captures(text)
.and_then(|cap| cap.get(1))
.and_then(|m| m.as_str().parse::<u32>().ok())
}
/// Extracts episode number from text using common patterns
fn extract_episode(text: &str) -> Option<u32> {
// Patterns: E02, e02, Episode 2, episode 2, x02, etc.
CONSTANTS.re_classifier_episode
.captures(text)
.and_then(|cap| cap.get(1))
.and_then(|m| m.as_str().parse::<u32>().ok())
}
/// Extracts show name from file path for series
/// Removes season/episode patterns and cleans up the name
pub fn extract_show_name(file: &ScannedMediaFile) -> (Option<Vec<MovieDbId>>, String) {
let file_name = Path::new(&file.file_name)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(&file.file_name);
let moviedb_ids: Vec<MovieDbId> = CONSTANTS.re_classifier_moviedb_id.captures_iter(file_name)
.filter_map(|caps| {
let id = caps[2].parse::<u32>().ok()?;
match &caps[1].to_lowercase()[..] {
"tmdb" => Some(MovieDbId::Tmdb(id)),
"tvdb" => Some(MovieDbId::Tvdb(id)),
_ => None,
}
})
.collect();
// Remove common series patterns
let cleaned = CONSTANTS.re_classifier_cleanup.replace(file_name, "").trim().to_string();
// Clean up remaining special characters
((!moviedb_ids.is_empty()).then_some(moviedb_ids), clear_filename(&cleaned))
}
/// Extracts movie title from file path
/// Attempts to extract tmdb-id, tvdb-id, year and remove quality tags
pub fn extract_movie_search_info(file: &ScannedMediaFile) -> (Option<Vec<MovieDbId>>, String, Option<u32>) {
let file_name = Path::new(&file.file_name)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(&file.file_name);
let moviedb_ids: Vec<MovieDbId> = CONSTANTS.re_classifier_moviedb_id.captures_iter(file_name)
.filter_map(|caps| {
let id = caps[2].parse::<u32>().ok()?;
match &caps[1].to_lowercase()[..] {
"tmdb" => Some(MovieDbId::Tmdb(id)),
"tvdb" => Some(MovieDbId::Tvdb(id)),
_ => None,
}
})
.collect();
// Try to extract year (4 digits in parentheses or standalone)
let year = CONSTANTS.re_classifier_year
.captures(file_name)
.and_then(|cap| cap.get(1))
.and_then(|m| m.as_str().parse::<u32>().ok())
.filter(|&y| (1900..=2100).contains(&y)); // Validate year range
// Remove year and everything after it, and quality tags
let mut title = file_name.to_string();
if let Some(mat) = CONSTANTS.re_classifier_year.find(file_name) {
title = file_name[..mat.start()].to_string();
}
title = clear_filename(&title);
((!moviedb_ids.is_empty()).then_some(moviedb_ids), title, year)
}
pub fn build_series_key(episodes: &[SeriesEpisodeFile]) -> SeriesKey {
if let Some(first) = episodes.first() {
if let Some((key, _season, _episode)) = Self::classify_series_file(&first.file) {
return key;
}
}
// fallback, not really helpful
SeriesKey {title: "Unknown".to_string(), tmdb_id: None, year: None}
}
}
@@ -256,72 +70,51 @@ mod tests {
}
}
#[test]
fn test_season_extraction() {
assert_eq!(MediaClassifier::extract_season("S01E02"), Some(1));
assert_eq!(MediaClassifier::extract_season("s03e05"), Some(3));
assert_eq!(MediaClassifier::extract_season("Season 2 Episode 1"), Some(2));
assert_eq!(MediaClassifier::extract_season("season_05_episode_03"), Some(5));
}
#[test]
fn test_episode_extraction() {
assert_eq!(MediaClassifier::extract_episode("S01E02"), Some(2));
assert_eq!(MediaClassifier::extract_episode("s03e05"), Some(5));
assert_eq!(MediaClassifier::extract_episode("Episode 12"), Some(12));
assert_eq!(MediaClassifier::extract_episode("1x15"), Some(15));
}
#[test]
fn test_extract_show_name() {
let file = create_test_file("Breaking.Bad.S01E01.mkv", "/tv/Breaking.Bad");
let (_, show_name) = MediaClassifier::extract_show_name(&file);
assert_eq!(show_name, "Breaking Bad");
let classification = MediaClassifier::classify(&file);
match classification {
MediaClassification::Movie { .. } => {
}
MediaClassification::Series { key, episode, season, metadata, .. } => {
assert_eq!(key.title, "Breaking Bad");
assert_eq!(episode, 1);
assert_eq!(season, 1);
assert_eq!(metadata.extension, Some("mkv".to_string()));
}
}
}
#[test]
fn test_extract_movie_title() {
let file = create_test_file("The.Matrix.1999.1080p.BluRay.mkv", "/movies");
let (_moviedb_id, title, year) = MediaClassifier::extract_movie_search_info(&file);
assert_eq!(title, "The Matrix");
assert_eq!(year, Some(1999));
let classification = MediaClassifier::classify(&file);
match classification {
MediaClassification::Movie { metadata } => {
assert_eq!(metadata.title, "The Matrix");
assert_eq!(metadata.year, Some(1999));
assert_eq!(metadata.extension, Some("mkv".to_string()));
}
MediaClassification::Series { .. } => {
}
}
}
#[test]
fn test_extract_movie_title_without_year() {
let file = create_test_file("Inception.1080p.BluRay.mkv", "/movies");
let (tmdbid, title, year) = MediaClassifier::extract_movie_search_info(&file);
assert_eq!(tmdbid, None);
assert_eq!(title, "Inception");
assert_eq!(year, None);
}
#[test]
fn test_classify_series() {
let series_pattern = Regex::new(r"(?i)s\d+e\d+").unwrap();
let classifier = MediaClassifier {
series_patterns: vec![series_pattern],
};
let file = create_test_file("Breaking.Bad.S01E01.mkv", "/tv");
let classification = classifier.classify(&file);
assert!(classification.is_series());
if let MediaClassification::Series { season, episode } = classification {
assert_eq!(season, Some(1));
assert_eq!(episode, Some(1));
let classification = MediaClassifier::classify(&file);
match classification {
MediaClassification::Movie { metadata } => {
assert_eq!(metadata.title, "Inception");
assert_eq!(metadata.year, None);
assert_eq!(metadata.extension, Some("mkv".to_string()));
}
MediaClassification::Series { .. } => {
}
}
}
#[test]
fn test_classify_movie() {
let series_pattern = Regex::new(r"(?i)s\d+e\d+").unwrap();
let classifier = MediaClassifier {
series_patterns: vec![series_pattern],
};
let file = create_test_file("The.Matrix.1999.mkv", "/movies");
let classification = classifier.classify(&file);
assert!(classification.is_movie());
}
}
backend/src/library/metadata.rs
+54 -44
View File
@@ -34,42 +34,42 @@ pub struct VideoClipMetadata {
pub struct MovieMetadata {
pub title: String,
// Original title (if different from title)
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub original_title: Option<String>,
// Release year
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub year: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub plot: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub tagline: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub runtime: Option<u32>,
// MPAA rating (e.g., "PG-13", "R")
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub mpaa: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub imdb_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub tmdb_id: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub tvdb_id: Option<u32>,
// Rating (0.0 - 10.0)
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub rating: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub genres: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub directors: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub writers: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub actors: Option<Vec<Actor>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub studios: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub poster: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub fanart: Option<String>,
pub source: MetadataSource,
pub last_updated: i64,
@@ -81,72 +81,82 @@ pub struct MovieMetadata {
pub struct SeriesMetadata {
pub title: String,
// Original title (if different)
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub original_title: Option<String>,
// First aired year
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub year: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub plot: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub mpaa: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub imdb_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub tmdb_id: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub tvdb_id: Option<u32>,
// Rating (0.0 - 10.0)
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub rating: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub genres: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub directors: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub writers: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub actors: Option<Vec<Actor>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub studios: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub poster: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub fanart: Option<String>,
// Status (e.g., "Continuing", "Ended")
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub status: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub seasons: Option<Vec<SeasonMetadata>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub episodes: Option<Vec<EpisodeMetadata>>,
#[serde(default)]
pub source: MetadataSource,
#[serde(default)]
pub number_of_episodes: u32,
#[serde(default)]
pub number_of_seasons: u32,
// Last updated timestamp (Unix epoch)
#[serde(default)]
pub last_updated: i64,
#[serde(default)]
pub videos: Option<Vec<VideoClipMetadata>>
}
// Episode metadata for TV series
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EpisodeMetadata {
#[serde(default)]
pub id: u32,
#[serde(default)]
pub tmdb_id: u32,
#[serde(default)]
pub title: String,
#[serde(default)]
pub season: u32,
#[serde(default)]
pub episode: u32,
// Aired date (ISO 8601 format)
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub aired: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub plot: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub runtime: Option<u32>,
// Rating (0.0 - 10.0)
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub rating: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub thumb: Option<String>,
pub file_path: String,
pub file_size: u64,
@@ -177,9 +187,9 @@ pub struct SeasonMetadata {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Actor {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub role: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub thumb: Option<String>,
}
@@ -224,8 +234,8 @@ impl MediaMetadata {
pub fn poster(&self) -> Option<&str> {
match self {
MediaMetadata::Movie(m) => m.poster.as_deref(),
MediaMetadata::Series(s) => s.poster.as_deref(),
MediaMetadata::Movie(m) => m.poster.as_deref().or(m.fanart.as_deref()),
MediaMetadata::Series(s) => s.poster.as_deref().or(s.fanart.as_deref()),
}
}
backend/src/library/metadata_resolver.rs
+61 -75
View File
@@ -1,12 +1,11 @@
use log::{debug, error, info, warn};
use crate::library::classifier::{MediaClassification, MediaClassifier};
use crate::library::metadata::{MediaMetadata, MetadataSource, MovieMetadata, SeriesMetadata};
use crate::library::nfo_reader::NfoReader;
use crate::library::scanner::ScannedMediaFile;
use crate::library::tmdb_client::TmdbClient;
use crate::library::{MediaGroup, MetadataStorage, MovieDbId, TMDB_API_KEY};
use crate::library::{MediaGroup, MetadataStorage, TMDB_API_KEY};
use crate::model::LibraryConfig;
use crate::ptt::{ptt_parse_title, PttMetadata};
// Metadata resolver that tries multiple sources to get video metadata
pub struct MetadataResolver {
@@ -35,17 +34,17 @@ impl MetadataResolver {
debug!("Resolving metadata for: {group}");
// Step 1: Classify the file
let (Some(file), classification) = (match group {
MediaGroup::Movie { file } => (Some(file), MediaClassification::Movie),
let (movie, Some(file), metadata) = (match group {
MediaGroup::Movie { file, metadata } => (true, Some(file), metadata),
MediaGroup::Series { show_key: _, episodes } => {
let episode = episodes.first()?;
(Some(&episode.file), MediaClassification::Series { season: None, episode: None })
},
(false, Some(&episode.file), &episode.metadata)
}
}) else { return None };
// Step 2: Try TMDB if enabled
if let Some(ref tmdb) = self.tmdb_client {
match self.resolve_from_tmdb(file, &classification, tmdb).await {
match self.resolve_from_tmdb(movie, file, metadata, tmdb).await {
Ok(Some(metadata)) => {
info!("Found TMDB metadata for: {}", file.file_path);
return Some(metadata);
@@ -58,18 +57,18 @@ impl MetadataResolver {
}
// TODO series implementation missing
if classification == MediaClassification::Movie {
// Step 3: Try to read existing NFO file
if let Some(metadata) = NfoReader::read_metadata(&file.path).await {
info!("Found NFO metadata for: {}", file.file_path);
return Some(metadata);
}
}
// if classification == MediaClassification::Movie {
// // Step 3: Try to read existing NFO file
// if let Some(metadata) = NfoReader::read_metadata(&file.path).await {
// info!("Found NFO metadata for: {}", file.file_path);
// return Some(metadata);
// }
// }
// Step 4: Fallback to filename parsing
if self.fallback_to_filename {
info!("Using filename-based metadata for: {}", file.file_path);
Some(Self::resolve_from_filename(file, &classification))
Some(Self::resolve_from_filename(movie, metadata))
} else {
warn!("No metadata found for: {}", file.file_path);
None
@@ -77,66 +76,48 @@ impl MetadataResolver {
}
// Attempts to resolve metadata from TMDB
async fn resolve_from_tmdb(&self, file: &ScannedMediaFile, classification: &MediaClassification, tmdb: &TmdbClient) -> Result<Option<MediaMetadata>, String> {
match classification {
MediaClassification::Movie => {
let (moviedb_ids, title, year) = MediaClassifier::extract_movie_search_info(file);
let tmdb_id = MovieDbId::get_tmdb_id(moviedb_ids.as_ref());
tmdb.search_movie(tmdb_id, &title, year).await
}
MediaClassification::Series { .. } => {
let (seriesdb_id, show_name) = MediaClassifier::extract_show_name(file);
let tmdb_id = MovieDbId::get_tmdb_id(seriesdb_id.as_ref());
debug!("Searching TMDB for series: {show_name}");
async fn resolve_from_tmdb(&self, movie: bool, file: &ScannedMediaFile, metadata: &PttMetadata, tmdb: &TmdbClient) -> Result<Option<MediaMetadata>, String> {
if movie {
tmdb.search_movie(metadata.tmdb, metadata.title.as_str(), metadata.year).await
} else {
let (series_year, tmdb_id) = if metadata.year.is_some() { (metadata.year, metadata.tmdb) } else {
// Try to extract year from parent directory if available
let year = file.path.parent()
file.path.parent()
.and_then(|p| p.file_name())
.and_then(|n| n.to_str())
.and_then(|s| {
// Look for 4-digit year in directory name
s.chars()
.collect::<Vec<_>>()
.windows(4)
.find_map(|w| {
let year_str: String = w.iter().collect();
year_str.parse::<u32>().ok()
.filter(|&y| (1900..=2100).contains(&y))
})
});
tmdb.search_series(tmdb_id, &show_name, year).await
}
.map_or((None, None), |s| {
let ptt = ptt_parse_title(s);
(ptt.year, ptt.tmdb)
})
};
tmdb.search_series(tmdb_id, metadata.title.as_str(), series_year).await
}
}
// Creates basic metadata from filename parsing
fn resolve_from_filename(
file: &ScannedMediaFile,
classification: &MediaClassification,
) -> MediaMetadata {
fn resolve_from_filename(movie: bool, metadata: &PttMetadata) -> MediaMetadata {
let timestamp = chrono::Utc::now().timestamp();
match classification {
MediaClassification::Movie => {
let (moviedb_id, title, year) = MediaClassifier::extract_movie_search_info(file);
MediaMetadata::Movie(MovieMetadata {
title,
year,
tmdb_id: MovieDbId::get_tmdb_id(moviedb_id.as_ref()),
tvdb_id: MovieDbId::get_tvdb_id(moviedb_id.as_ref()),
source: MetadataSource::FilenameParsed,
last_updated: timestamp,
..MovieMetadata::default()
})
}
MediaClassification::Series { season: _, episode: _ } => {
let (_moviedb_id, show_name) = MediaClassifier::extract_show_name(file);
MediaMetadata::Series(SeriesMetadata {
title: show_name,
source: MetadataSource::FilenameParsed,
last_updated: timestamp,
..SeriesMetadata::default()
})
}
if movie {
MediaMetadata::Movie(MovieMetadata {
title: metadata.title.clone(),
year: metadata.year,
tmdb_id: metadata.tmdb,
tvdb_id: metadata.tvdb,
source: MetadataSource::FilenameParsed,
last_updated: timestamp,
..MovieMetadata::default()
})
} else {
MediaMetadata::Series(SeriesMetadata {
title: metadata.title.clone(),
year: metadata.year,
tmdb_id: metadata.tmdb,
tvdb_id: metadata.tvdb,
source: MetadataSource::FilenameParsed,
last_updated: timestamp,
..SeriesMetadata::default()
})
}
}
}
@@ -144,9 +125,10 @@ impl MetadataResolver {
#[cfg(test)]
mod tests {
use super::*;
use crate::model::{LibraryClassificationConfig, LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig};
use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig};
use std::path::PathBuf;
use std::time::Duration;
use crate::library::{MediaClassification, MediaClassifier};
fn create_test_config(tmdb_enabled: bool) -> LibraryConfig {
LibraryConfig {
@@ -174,10 +156,6 @@ mod tests {
fallback_to_filename: true,
formats: vec![],
},
classification: LibraryClassificationConfig {
series_patterns: vec![],
series_directory_patterns: vec![],
},
playlist: LibraryPlaylistConfig {
movie_category: "Movies".to_string(),
series_category: "Series".to_string(),
@@ -206,7 +184,11 @@ mod tests {
.unwrap_or_else(|_| reqwest::Client::new());
let resolver = MetadataResolver::from_config(&config, client, MetadataStorage::new(PathBuf::from("/tmp")));
let file = create_test_file("The.Matrix.1999.1080p.mkv");
let group = MediaGroup::Movie { file };
let metadata = match MediaClassifier::classify(&file) {
MediaClassification::Movie { metadata, .. } => metadata,
MediaClassification::Series { metadata, .. } => metadata,
};
let group = MediaGroup::Movie { file, metadata: Box::new(metadata) };
let metadata = resolver.resolve(&group).await;
assert!(metadata.is_some());
@@ -214,7 +196,7 @@ mod tests {
if let Some(MediaMetadata::Movie(movie)) = metadata {
assert_eq!(movie.title, "The Matrix");
assert_eq!(movie.year, Some(1999));
assert_eq!(movie.source, MetadataSource::FilenameParsed);
assert_eq!(movie.source, MetadataSource::Tmdb);
} else {
panic!("Expected movie metadata");
}
@@ -231,7 +213,11 @@ mod tests {
config.metadata.fallback_to_filename = false;
let resolver = MetadataResolver::from_config(&config, client, MetadataStorage::new(PathBuf::from("/tmp")));
let file = create_test_file("Unknown.Movie.mkv");
let group = MediaGroup::Movie { file };
let metadata = match MediaClassifier::classify(&file) {
MediaClassification::Movie { metadata, .. } => metadata,
MediaClassification::Series { metadata, .. } => metadata,
};
let group = MediaGroup::Movie { file, metadata: Box::new(metadata) };
let metadata = resolver.resolve(&group).await;
assert!(metadata.is_none());
backend/src/library/processor.rs
+3 -3
View File
@@ -96,7 +96,7 @@ impl LibraryProcessor {
let scanned_paths: std::collections::HashSet<_> = media_groups
.iter()
.flat_map(|group| match group {
MediaGroup::Movie { file } => vec![file.file_path.as_str()],
MediaGroup::Movie { file, .. } => vec![file.file_path.as_str()],
MediaGroup::Series { episodes, .. } => episodes.iter().map(|ep| ep.file.file_path.as_str()).collect(),
})
.collect();
@@ -118,7 +118,7 @@ impl LibraryProcessor {
async fn process_group(&self, group: &MediaGroup, existing_map: &HashMap<String, MetadataCacheEntry>, force_rescan: bool) -> Result<ProcessAction, String> {
match group {
MediaGroup::Movie { file: _ } => {
MediaGroup::Movie { file: _, .. } => {
self.process_movie(group, existing_map, force_rescan).await
}
MediaGroup::Series { show_key: _, episodes: _ } => {
@@ -130,7 +130,7 @@ impl LibraryProcessor {
// Processes a single video file
async fn process_movie(&self, group: &MediaGroup, existing_map: &HashMap<String, MetadataCacheEntry>, force_rescan: bool,
) -> Result<ProcessAction, String> {
let MediaGroup::Movie { file } = group else { return Err(format!("Expected movie to resolve but got {group}")) };
let MediaGroup::Movie { file, .. } = group else { return Err(format!("Expected movie to resolve but got {group}")) };
// Check if file already exists in cache
let (cache_entry, status) = if let Some(existing_entry) = existing_map.get(&file.file_path) {
// Check if file has been modified
backend/src/library/scanner.rs
+22 -18
View File
@@ -1,4 +1,4 @@
use crate::library::MediaClassifier;
use crate::library::{MediaClassification, MediaClassifier};
use crate::model::{LibraryConfig, LibraryScanDirectory};
use log::{debug, error, info, trace, warn};
use std::collections::HashMap;
@@ -6,6 +6,7 @@ use std::fmt::{Display, Formatter};
use std::path::{Path, PathBuf};
use tokio::fs;
use tokio::io;
use crate::ptt::PttMetadata;
#[derive(Hash, Eq, PartialEq, Clone, Debug)]
pub struct SeriesKey {
@@ -30,6 +31,7 @@ impl Display for SeriesKey {
pub enum MediaGroup {
Movie {
file: ScannedMediaFile,
metadata: Box<PttMetadata>,
},
Series {
show_key: SeriesKey,
@@ -40,7 +42,7 @@ pub enum MediaGroup {
impl Display for MediaGroup {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
MediaGroup::Movie { file } => write!(f, "{}", file.file_path),
MediaGroup::Movie { file, .. } => write!(f, "{}", file.file_path),
MediaGroup::Series { show_key, .. } => write!(f, "{}", show_key.title),
}
}
@@ -51,6 +53,7 @@ pub struct SeriesEpisodeFile {
pub file: ScannedMediaFile,
pub season: u32,
pub episode: u32,
pub metadata: Box<PttMetadata>,
}
pub struct MediaGrouper;
@@ -61,17 +64,22 @@ impl MediaGrouper {
let mut movies = Vec::new();
for file in files {
if let Some((key, season, episode)) = MediaClassifier::classify_series_file(&file) {
series_map
.entry(key)
.or_default()
.push(SeriesEpisodeFile {
file,
season,
episode,
});
} else {
movies.push(MediaGroup::Movie { file });
let classification = MediaClassifier::classify(&file);
match classification {
MediaClassification::Movie { metadata } => {
movies.push(MediaGroup::Movie { file, metadata: Box::new(metadata) });
}
MediaClassification::Series { key, metadata, season, episode,.. } => {
series_map
.entry(key)
.or_default()
.push(SeriesEpisodeFile {
file,
season,
episode,
metadata: Box::new(metadata),
});
}
}
}
@@ -274,7 +282,7 @@ impl LibraryScanner {
#[cfg(test)]
mod tests {
use super::*;
use crate::model::{LibraryClassificationConfig, LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig};
use crate::model::{LibraryMetadataConfig, LibraryMetadataReadConfig, LibraryPlaylistConfig, LibraryTmdbConfig};
fn create_test_config() -> LibraryConfig {
LibraryConfig {
@@ -302,10 +310,6 @@ mod tests {
fallback_to_filename: true,
formats: vec![],
},
classification: LibraryClassificationConfig {
series_patterns: vec![],
series_directory_patterns: vec![],
},
playlist: LibraryPlaylistConfig {
movie_category: "Local Movies".to_string(),
series_category: "Local Series".to_string(),
backend/src/library/tmdb_client.rs
+2
View File
@@ -100,6 +100,8 @@ impl TmdbClient {
// Searches for a TV series by title and optional year
pub async fn search_series(&self, tmdb_id: Option<u32>, title: &str, year: Option<u32>) -> Result<Option<MediaMetadata>, String> {
debug!("Searching TMDB for series: {title}");
let key = format!("{title}-{tmdb_id:?}-{year:?}");
if self.fetched_series_key.read().await.contains(&key) {
backend/src/main.rs
+12 -3
View File
@@ -14,7 +14,7 @@ use crate::model::{
AppConfig, Config, Healthcheck, HealthcheckConfig, ProcessTargets, SourcesConfig,
};
use crate::processing::processor::playlist;
use crate::utils::init_logger;
use crate::utils::{db_viewer, init_logger};
use crate::utils::request::create_client;
use crate::utils::{config_file_reader, resolve_env_var};
use crate::library::LibraryProcessor;
@@ -32,7 +32,7 @@ use std::sync::Arc;
#[command(name = "tuliprox")]
#[command(author = "euzu <euzu@proton.me>")]
#[command(version)]
#[command(about = "Extended M3U playlist filter", long_about = None)]
#[command(about = "Extended playlist proxy", long_about = None)]
struct Args {
/// The config directory
#[arg(short = 'p', long = "config-path")]
@@ -85,6 +85,13 @@ struct Args {
/// Force rescan of all local Library files
#[arg(long = "force-library-rescan", default_value_t = false, default_missing_value = "true")]
force_library_rescan: bool,
#[arg(long = "dbx")]
db_xtream_file_name: Option<String>,
#[arg(long = "dbm")]
db_m3u_file_name: Option<String>,
}
const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -102,6 +109,8 @@ const BUILD_TIMESTAMP: &str = env!("VERGEN_BUILD_TIMESTAMP");
async fn main() {
let args = Args::parse();
db_viewer(args.db_xtream_file_name.as_deref(), args.db_m3u_file_name.as_deref());
if args.genpwd {
match generate_password() {
Ok(pwd) => println!("{pwd}"),
@@ -113,7 +122,7 @@ async fn main() {
let mut config_paths = get_file_paths(&args);
init_logger(
args.log_level.as_ref(),
args.log_level.as_deref(),
config_paths.config_file_path.as_str(),
);
backend/src/model/config/api_proxy.rs
+1 -1
View File
@@ -128,7 +128,7 @@ impl ApiProxyConfig {
}
} else {
let user_db_path = get_api_user_db_path(cfg);
if user_db_path.exists() {
if let Ok(true) = tokio::fs::try_exists(&user_db_path).await {
// we cant have user defined in db file.
// we need to load them and save them into the config file
if let Ok(stored_users) = load_api_user(cfg).await {
backend/src/model/config/library.rs
-44
View File
@@ -1,6 +1,4 @@
use crate::model::macros;
use log::error;
use regex::Regex;
use shared::error::TuliproxError;
use shared::model::{default_metadata_path, LibraryConfigDto, LibraryContentType, LibraryMetadataFormat};
use std::path::PathBuf;
@@ -39,12 +37,6 @@ pub struct LibraryTmdbConfig {
pub language: String,
}
#[derive(Debug, Clone)]
pub struct LibraryClassificationConfig {
pub series_patterns: Vec<Regex>,
pub series_directory_patterns: Vec<Regex>,
}
#[derive(Debug, Clone)]
pub struct LibraryPlaylistConfig {
pub movie_category: String,
@@ -58,7 +50,6 @@ pub struct LibraryConfig {
pub scan_directories: Vec<LibraryScanDirectory>,
pub supported_extensions: Vec<String>,
pub metadata: LibraryMetadataConfig,
pub classification: LibraryClassificationConfig,
pub playlist: LibraryPlaylistConfig,
}
@@ -93,37 +84,6 @@ macros::from_impl!(LibraryConfig);
impl From<&LibraryConfigDto> for LibraryConfig {
fn from(dto: &LibraryConfigDto) -> Self {
// Compile series patterns
let series_patterns = dto
.classification
.series_patterns
.iter()
.filter_map(|pattern| {
match Regex::new(pattern) {
Ok(re) => Some(re),
Err(e) => {
error!("Invalid series pattern '{pattern}': {e}");
None
}
}
})
.collect();
// Compile directory patterns
let series_directory_patterns = dto
.classification
.series_directory_patterns
.iter()
.filter_map(|pattern| {
match Regex::new(pattern) {
Ok(re) => Some(re),
Err(e) => {
error!("Invalid series directory pattern '{pattern}': {e}");
None
}
}
})
.collect();
Self {
enabled: dto.enabled,
@@ -159,10 +119,6 @@ impl From<&LibraryConfigDto> for LibraryConfig {
fallback_to_filename: dto.metadata.fallback_to_filename,
formats: dto.metadata.formats.clone(),
},
classification: LibraryClassificationConfig {
series_patterns,
series_directory_patterns,
},
playlist: LibraryPlaylistConfig {
movie_category: dto.playlist.movie_category.clone(),
series_category: dto.playlist.series_category.clone(),
backend/src/model/config/log.rs
+2 -2
View File
@@ -14,13 +14,13 @@ pub struct LogConfig {
pub sanitize_sensitive_info: bool,
#[serde(default)]
pub log_active_user: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub log_level: Option<String>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)]
pub struct LogLevelConfig {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub log: Option<LogConfig>,
}
backend/src/model/config/target.rs
+3
View File
@@ -283,6 +283,9 @@ impl ConfigTarget {
}
pub fn is_force_redirect(&self, item_type: PlaylistItemType) -> bool {
if item_type.is_local() {
return false;
}
self.options
.as_ref()
.and_then(|options| options.force_redirect.as_ref())
backend/src/model/healthcheck.rs
+1 -1
View File
@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
pub struct Healthcheck {
pub status: String,
pub version: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub build_time: Option<String>,
pub server_time: String,
}
backend/src/model/media_properties.rs
+85 -70
View File
@@ -1,8 +1,6 @@
// backend/src/model/media_properties.rs
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use std::fmt;
use serde_json::{Value, Map};
// Enum for Video Resolution
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
@@ -161,67 +159,12 @@ impl MediaQuality {
parts.join(separator)
}
/// Extracts media quality information from an `ffprobe` info block.
/// The `info_block` is expected to be a `serde_json::Value` object.
pub fn from_ffprobe_info(info_block: &Value) -> Option<Self> {
let video_info = info_block.get("video")?.as_object()?;
fn from_ffprobe_info_audio(audio: Option<&str>) -> Option<(AudioCodec, AudioChannels)> {
// Assuming the first audio stream is the primary one.
let audio_info = info_block.get("audio")?.as_object()?;
let audio_info = audio.and_then(|v| serde_json::from_str::<Map<String, Value>>(v).ok())?;
// Helper to get a value by trying a prioritized list of field names.
let get_value = |obj: &Map<String, Value>, fields: &[&str]| -> Option<Value> {
for field in fields {
if let Some(value) = obj.get(*field) {
if !value.is_null() {
return Some(value.clone());
}
}
}
None
};
// 1. Classify video resolution from width
let resolution = get_value(video_info, &["height", "coded_height"])
.and_then(|v| v.as_u64())
.map_or(VideoResolution::default(), |h| match h {
_ if h >= 4300 => VideoResolution::P4320,
_ if h >= 2100 => VideoResolution::P2160,
_ if h >= 1000 => VideoResolution::P1080,
_ if h >= 700 => VideoResolution::P720,
_ => VideoResolution::SD,
});
// 2. Classify video codec
let video_codec = get_value(video_info, &["codec_name"])
.and_then(|v| v.as_str().map(str::to_lowercase))
.map_or(VideoCodec::default(), |name| match name.as_str() {
"h264" => VideoCodec::H264,
"hevc" => VideoCodec::H265,
"mpeg4" => VideoCodec::MPEG4,
"vc1" => VideoCodec::VC1,
_ => VideoCodec::default(),
});
// 3. Classify dynamic range
let dynamic_range = {
let tag_string = get_value(video_info, &["codec_tag_string"])
.and_then(|v| v.as_str().map(str::to_lowercase));
if tag_string == Some("dovi".to_string()) {
VideoDynamicRange::DV
} else {
get_value(video_info, &["color_transfer"])
.and_then(|v| v.as_str().map(str::to_lowercase))
.map_or(VideoDynamicRange::SDR, |ct| match ct.as_str() {
"smpte2084" => VideoDynamicRange::HDR10,
"arib-std-b67" => VideoDynamicRange::HLG,
_ => VideoDynamicRange::SDR,
})
}
};
// 4. Classify audio codec
let audio_codec = get_value(audio_info, &["codec_name"])
let audio_codec = get_value(&audio_info, &["codec_name"])
.and_then(|v| v.as_str().map(str::to_lowercase))
.map_or(AudioCodec::default(), |name| match name.as_str() {
"aac" => AudioCodec::AAC,
@@ -234,7 +177,7 @@ impl MediaQuality {
});
// 5. Classify audio channels
let audio_channels = get_value(audio_info, &["channel_layout"])
let audio_channels = get_value(&audio_info, &["channel_layout"])
.and_then(|v| v.as_str().map(str::to_lowercase))
.map_or(AudioChannels::default(), |layout| match layout.as_str() {
l if l.starts_with("7.1") => AudioChannels::Surround71,
@@ -244,12 +187,84 @@ impl MediaQuality {
_ => AudioChannels::default(),
});
Some(Self {
resolution,
video_codec,
dynamic_range,
audio_codec,
audio_channels,
})
Some((audio_codec, audio_channels))
}
}
fn from_ffprobe_info_video(video: Option<&str>) -> Option<(VideoResolution, VideoCodec, VideoDynamicRange)> {
let video_info = video.and_then(|v| serde_json::from_str::<Map<String, Value>>(v).ok())?;
// 1. Classify video resolution from width
let resolution = get_value(&video_info, &["height", "coded_height"])
.and_then(|v| v.as_u64())
.map_or(VideoResolution::default(), |h| match h {
_ if h >= 4300 => VideoResolution::P4320,
_ if h >= 2100 => VideoResolution::P2160,
_ if h >= 1000 => VideoResolution::P1080,
_ if h >= 700 => VideoResolution::P720,
_ => VideoResolution::SD,
});
// 2. Classify video codec
let video_codec = get_value(&video_info, &["codec_name"])
.and_then(|v| v.as_str().map(str::to_lowercase))
.map_or(VideoCodec::default(), |name| match name.as_str() {
"h264" => VideoCodec::H264,
"hevc" => VideoCodec::H265,
"mpeg4" => VideoCodec::MPEG4,
"vc1" => VideoCodec::VC1,
_ => VideoCodec::default(),
});
// 3. Classify dynamic range
let dynamic_range = {
let tag_string = get_value(&video_info, &["codec_tag_string"])
.and_then(|v| v.as_str().map(str::to_lowercase));
if tag_string == Some("dovi".to_string()) {
VideoDynamicRange::DV
} else {
get_value(&video_info, &["color_transfer"])
.and_then(|v| v.as_str().map(str::to_lowercase))
.map_or(VideoDynamicRange::SDR, |ct| match ct.as_str() {
"smpte2084" => VideoDynamicRange::HDR10,
"arib-std-b67" => VideoDynamicRange::HLG,
_ => VideoDynamicRange::SDR,
})
}
};
Some((resolution, video_codec, dynamic_range))
}
/// Extracts media quality information from an `ffprobe` info block.
/// The `info_block` is expected to be a `serde_json::Value` object.
pub fn from_ffprobe_info(audio: Option<&str>, video: Option<&str>) -> Option<Self> {
match (Self::from_ffprobe_info_video(video), Self::from_ffprobe_info_audio(audio)) {
(Some((resolution, video_codec, dynamic_range)),
Some((audio_codec, audio_channels))) => {
Some(Self {
resolution,
video_codec,
dynamic_range,
audio_codec,
audio_channels,
})
}
// TODO to restrictive, maybe only video or audio could be processed
_ => None,
}
}
}
// Helper to get a value by trying a prioritized list of field names.
fn get_value(obj: &Map<String, Value>, fields: &[&str]) -> Option<Value> {
for field in fields {
if let Some(value) = obj.get(*field) {
if !value.is_null() {
return Some(value.clone());
}
}
}
None
}
backend/src/model/playlist_categories.rs
+3 -3
View File
@@ -1,9 +1,9 @@
#[derive(Debug, serde::Serialize, serde::Deserialize, Default)]
pub struct PlaylistCategories {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub live: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub vod: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub series: Option<Vec<String>>,
}
backend/src/model/xtream.rs
+25 -694
View File
@@ -1,68 +1,10 @@
use std::collections::BTreeMap;
use crate::model::{AppConfig, ProxyUserCredentials};
use crate::model::{ConfigTarget, XtreamTargetOutput};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{Value};
use shared::model::{xtream_const, PlaylistItem, ProxyUserStatus, XtreamPlaylistItem};
use shared::model::{ClusterFlags, PlaylistEntry, XtreamCluster};
use shared::utils::{deserialize_as_option_string, deserialize_as_string, deserialize_as_string_array, deserialize_number_from_string,
get_non_empty_str, opt_string_or_number_u32, string_default_on_null, string_or_number_f64, string_or_number_u32};
use std::iter::FromIterator;
use serde::ser::SerializeMap;
use serde_json::value::RawValue;
macro_rules! add_str_property_if_exists {
($vec:expr, $prop:expr, $prop_name:expr) => {
$vec.insert(String::from($prop_name), Value::String($prop.to_string()));
}
}
macro_rules! add_rc_str_property_if_exists {
($vec:expr, $prop:expr, $prop_name:expr) => {
if let Some(v) = $prop.as_ref() {
$vec.insert(String::from($prop_name), Value::String(v.to_string()));
}
}
}
macro_rules! add_opt_i64_property_if_exists {
($vec:expr, $prop:expr, $prop_name:expr) => {
if let Some(v) = $prop.as_ref() {
$vec.insert(String::from($prop_name), Value::Number(serde_json::value::Number::from(i64::from(*v))));
}
}
}
macro_rules! add_opt_f64_property_if_exists {
($vec:expr, $prop:expr, $prop_name:expr) => {
$prop.as_ref().map(|v| $vec.insert(String::from($prop_name), Value::Number(serde_json::value::Number::from_f64(f64::from(*v)).unwrap_or_else(|| serde_json::Number::from(0)))));
}
}
macro_rules! add_f64_property_if_exists {
($vec:expr, $prop:expr, $prop_name:expr) => {
$vec.insert(String::from($prop_name), Value::Number(serde_json::value::Number::from_f64(f64::from($prop)).unwrap_or_else(|| serde_json::Number::from(0))));
}
}
macro_rules! add_i64_property_if_exists {
($vec:expr, $prop:expr, $prop_name:expr) => {
$vec.insert(String::from($prop_name), Value::Number(serde_json::value::Number::from(i64::from($prop))));
}
}
macro_rules! add_to_doc_str_property_if_not_exists {
($document:expr, $prop_name:expr, $prop_value:expr) => {
match $document.get($prop_name) {
None => {
$document.insert(String::from($prop_name), $prop_value);
}
Some(value) => { if Value::is_null(value) {
$document.insert(String::from($prop_name), $prop_value);
}}
}
}
}
use serde::{Deserialize, Serialize};
use serde_json::Value;
use shared::model::{PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, ProxyUserStatus, XtreamMappingOptions};
use shared::utils::{deserialize_as_string, deserialize_number_from_string_or_zero};
use enum_iterator::all;
#[derive(Debug, Default)]
pub struct XtreamLoginInfo {
@@ -72,8 +14,8 @@ pub struct XtreamLoginInfo {
#[derive(Deserialize, Default)]
pub struct XtreamCategory {
#[serde(deserialize_with = "deserialize_as_string")]
pub category_id: String,
#[serde(deserialize_with = "deserialize_number_from_string_or_zero")]
pub category_id: u32,
#[serde(deserialize_with = "deserialize_as_string")]
pub category_name: String,
//pub parent_id: i32,
@@ -87,492 +29,30 @@ impl XtreamCategory {
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct XtreamStream {
#[serde(default, deserialize_with = "deserialize_as_string")]
pub name: String,
#[serde(default, deserialize_with = "deserialize_as_string")]
pub category_id: String,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub stream_id: Option<u32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub series_id: Option<u32>,
#[serde(default, deserialize_with = "deserialize_as_string")]
pub stream_icon: String,
#[serde(default, deserialize_with = "deserialize_as_string")]
pub direct_source: String,
// optional attributes
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub custom_sid: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_string_array")]
pub backdrop_path: Option<Vec<String>>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub added: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub cast: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub container_extension: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub cover: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub director: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub episode_run_time: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub genre: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub last_modified: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub plot: Option<String>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub rating: Option<f64>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub rating_5based: Option<f64>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub release_date: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub stream_type: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub title: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub year: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub trailer: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub youtube_trailer: Option<String>,
#[serde(default, deserialize_with = "deserialize_as_option_string")]
pub epg_channel_id: Option<String>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub tv_archive: Option<i32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub tv_archive_duration: Option<i32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub tmdb: Option<u32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub tmdb_id: Option<u32>,
#[serde(default, deserialize_with = "deserialize_number_from_string")]
pub is_adult: Option<i32>,
pub fn xtream_mapping_option_from_target_options(target: &ConfigTarget, target_output: &XtreamTargetOutput,
cfg: &AppConfig, user: &ProxyUserCredentials,
resource_url: Option<&str>) -> XtreamMappingOptions {
}
let force_redirect = target.options.as_ref().and_then(|o| o.force_redirect);
let mut reverse_item_types = PlaylistItemTypeSet::empty();
impl XtreamStream {
pub fn get_stream_id(&self) -> u32 {
self.stream_id.unwrap_or_else(|| self.series_id.unwrap_or(0))
}
pub fn get_additional_properties(&self) -> Option<Box<RawValue>> {
let mut result = serde_json::Map::new();
if let Some(bdpath) = self.backdrop_path.as_ref() {
if !bdpath.is_empty() {
result.insert(String::from(xtream_const::XC_PROP_BACKDROP_PATH), Value::Array(Vec::from([Value::String(String::from(bdpath.first()?))])));
}
}
add_rc_str_property_if_exists!(result, self.tmdb_id.or(self.tmdb), "tmdb");
add_rc_str_property_if_exists!(result, self.added, "added");
add_rc_str_property_if_exists!(result, self.cast, "cast");
add_rc_str_property_if_exists!(result, self.container_extension, "container_extension");
add_rc_str_property_if_exists!(result, self.cover, "cover");
add_rc_str_property_if_exists!(result, self.director, "director");
add_rc_str_property_if_exists!(result, self.episode_run_time, "episode_run_time");
add_rc_str_property_if_exists!(result, self.genre, "genre");
add_rc_str_property_if_exists!(result, self.last_modified, "last_modified");
add_rc_str_property_if_exists!(result, self.plot, "plot");
add_opt_f64_property_if_exists!(result, self.rating, "rating");
add_opt_f64_property_if_exists!(result, self.rating_5based, "rating_5based");
add_rc_str_property_if_exists!(result, self.release_date, "release_date");
add_rc_str_property_if_exists!(result, self.stream_type, "stream_type");
add_rc_str_property_if_exists!(result, self.title, "title");
add_rc_str_property_if_exists!(result, self.year, "year");
add_rc_str_property_if_exists!(result, self.trailer, "trailer");
add_rc_str_property_if_exists!(result, self.youtube_trailer, "youtube_trailer");
add_rc_str_property_if_exists!(result, self.epg_channel_id, "epg_channel_id");
add_opt_i64_property_if_exists!(result, self.tv_archive, "tv_archive");
add_opt_i64_property_if_exists!(result, self.tv_archive_duration, "tv_archive_duration");
add_opt_i64_property_if_exists!(result, self.is_adult, "is_adult");
if result.is_empty() {
None
} else {
let s = serde_json::to_string(&Value::Object(result)).ok()?;
RawValue::from_string(s).ok()
for item_type in all::<PlaylistItemType>() {
if user.proxy.is_reverse(item_type) && !force_redirect.as_ref().is_some_and(|o| o.has_cluster(item_type)) {
reverse_item_types.insert(item_type);
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamMovieInfo {
pub info: XtreamMovieInfoDetails,
pub movie_data: XtreamMovieData,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamMovieInfoDetails {
pub kinopoisk_url: Option<String>,
pub tmdb_id: Option<String>,
pub name: Option<String>,
pub o_name: Option<String>,
pub cover_big: Option<String>,
pub movie_image: Option<String>,
pub releasedate: Option<String>,
pub episode_run_time: Option<u32>,
pub youtube_trailer: Option<String>,
pub director: Option<String>,
pub actors: Option<String>,
pub cast: Option<String>,
pub description: Option<String>,
pub plot: Option<String>,
pub age: Option<String>,
pub mpaa_rating: Option<String>,
#[serde(default)]
pub rating_count_kinopoisk: u32,
pub country: Option<String>,
pub genre: Option<String>,
#[serde(default)]
pub backdrop_path: Vec<String>,
pub duration_secs: Option<String>,
pub duration: Option<String>,
#[serde(default)]
pub video: Vec<XtreamStreamInfo>,
#[serde(default)]
pub audio: Vec<XtreamStreamInfo>,
#[serde(default)]
pub bitrate: u32,
pub rating: Option<String>,
pub runtime: Option<String>,
pub status: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamStreamInfo {
#[serde(flatten)]
pub extra: std::collections::HashMap<String, serde_json::Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamMovieData {
pub stream_id: u32,
pub name: String,
pub added: Option<String>,
pub category_id: u32,
#[serde(default)]
pub category_ids: Vec<u32>,
pub container_extension: Option<String>,
pub custom_sid: Option<String>,
pub direct_source: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamSeriesInfoSeason {
#[serde(default, deserialize_with = "string_default_on_null")]
pub air_date: String,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub episode_count: u32,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub id: u32,
#[serde(default, deserialize_with = "string_default_on_null")]
pub name: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub overview: String,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub season_number: u32,
#[serde(default, deserialize_with = "string_or_number_f64")]
pub vote_average: f64,
#[serde(default, deserialize_with = "string_default_on_null")]
pub cover: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub cover_big: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[allow(non_snake_case)]
pub struct XtreamSeriesInfoInfo {
#[serde(default, deserialize_with = "string_default_on_null")]
pub(crate) name: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub cover: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub plot: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub cast: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub director: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub genre: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub release_date: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub releaseDate: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub releasedate: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub last_modified: String,
#[serde(default, deserialize_with = "string_or_number_f64")]
pub rating: f64,
#[serde(default, deserialize_with = "string_or_number_f64")]
pub rating_5based: f64,
#[serde(default, deserialize_with = "deserialize_as_string_array")]
pub backdrop_path: Option<Vec<String>>,
#[serde(default, deserialize_with = "string_default_on_null")]
pub trailer: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub youtube_trailer: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub episode_run_time: String,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub category_id: u32,
#[serde(default, deserialize_with = "opt_string_or_number_u32")]
pub tmdb_id: Option<u32>,
#[serde(default, deserialize_with = "opt_string_or_number_u32")]
pub tmdb: Option<u32>,
#[serde(default, deserialize_with = "opt_string_or_number_u32")]
pub year: Option<u32>,
}
#[allow(non_snake_case)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct XtreamSeriesInfoEpisodeInfo {
#[serde(default, deserialize_with = "opt_string_or_number_u32")]
pub id: Option<u32>,
#[serde(default, deserialize_with = "opt_string_or_number_u32")]
pub tmdb_id: Option<u32>,
#[serde(default, deserialize_with = "opt_string_or_number_u32")]
pub tmdb: Option<u32>,
#[serde(default, deserialize_with = "string_default_on_null")]
pub release_date: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub releaseDate: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub releasedate: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub plot: String,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub duration_secs: u32,
#[serde(default, deserialize_with = "string_default_on_null")]
pub duration: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub movie_image: String,
#[serde(default)]
pub video: Option<Box<RawValue>>,
#[serde(default)]
pub audio: Option<Box<RawValue>>,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub bitrate: u32,
#[serde(default, deserialize_with = "string_or_number_f64")]
pub rating: f64,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub season: u32,
}
// Used for serde_json deserialization, can not be used with bincode
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamSeriesInfoEpisode {
#[serde(default, deserialize_with = "string_or_number_u32")]
pub id: u32,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub episode_num: u32,
#[serde(default, deserialize_with = "string_default_on_null")]
pub title: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub container_extension: String,
#[serde(default)]
pub info: Option<XtreamSeriesInfoEpisodeInfo>,
#[serde(default, deserialize_with = "string_default_on_null")]
pub custom_sid: String,
#[serde(default, deserialize_with = "string_default_on_null")]
pub added: String,
#[serde(default, deserialize_with = "string_or_number_u32")]
pub season: u32,
#[serde(default, deserialize_with = "string_default_on_null")]
pub direct_source: String,
}
impl XtreamSeriesInfoEpisode {
pub fn get_id(&self) -> u32 {
self.id
}
}
//bincode does not support deserialize_with. We use this struct for db
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamSeriesEpisode {
pub id: u32,
pub episode_num: u32,
pub title: String,
pub container_extension: String,
pub custom_sid: String,
pub added: String,
pub season: u32,
pub tmdb_id: u32,
pub direct_source: String,
}
impl XtreamSeriesEpisode {
pub fn from(info_episode: &XtreamSeriesInfoEpisode) -> Self {
Self {
id: info_episode.get_id(),
episode_num: info_episode.episode_num,
title: info_episode.title.clone(),
container_extension: info_episode.container_extension.clone(),
custom_sid: info_episode.custom_sid.clone(),
added: info_episode.added.clone(),
season: info_episode.season,
tmdb_id: info_episode.info.as_ref().and_then(|info| info.tmdb_id.or(info.tmdb)).unwrap_or(0),
direct_source: info_episode.direct_source.clone(),
}
}
}
// sometimes episodes are a map with season as key, sometimes an array
fn deserialize_episodes<'de, D>(deserializer: D) -> Result<Option<Vec<XtreamSeriesInfoEpisode>>, D::Error>
where
D: Deserializer<'de>,
{
// read as generic value
let value = Value::deserialize(deserializer)?;
match value {
Value::Null => Ok(None),
Value::Array(array) => {
if array.is_empty() {
Ok(None)
} else {
let mut result = Vec::new();
for inner in array {
if let Some(inner_array) = inner.as_array() {
for item in inner_array {
let ep: XtreamSeriesInfoEpisode = serde_json::from_value(item.clone())
.map_err(serde::de::Error::custom)?;
result.push(ep);
}
}
}
Ok(Some(result))
}
}
Value::Object(object) => {
if object.is_empty() {
Ok(None)
} else {
let mut result = Vec::new();
for (_key, val) in object {
if let Some(inner_array) = val.as_array() {
for item in inner_array {
let ep: XtreamSeriesInfoEpisode = serde_json::from_value(item.clone())
.map_err(serde::de::Error::custom)?;
result.push(ep);
}
}
}
Ok(Some(result))
}
}
_ => Err(serde::de::Error::custom("Invalid format for episodes")),
}
}
#[allow(clippy::ref_option)]
fn serialize_episodes<S>(
episodes: &Option<Vec<XtreamSeriesInfoEpisode>>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match episodes {
None => {
let map = serializer.serialize_map(Some(0))?;
map.end()
},
Some(list) => {
if list.is_empty() {
let map = serializer.serialize_map(Some(0))?;
return map.end();
}
let mut seasons: BTreeMap<String, Vec<&XtreamSeriesInfoEpisode>> = BTreeMap::new();
for ep in list {
seasons.entry(ep.season.to_string()).or_default().push(ep);
}
seasons.serialize(serializer)
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XtreamSeriesInfo {
#[serde(default)]
pub seasons: Option<Vec<XtreamSeriesInfoSeason>>,
#[serde(default)]
pub info: Option<XtreamSeriesInfoInfo>,
#[serde(default, serialize_with = "serialize_episodes", deserialize_with = "deserialize_episodes")]
pub episodes: Option<Vec<XtreamSeriesInfoEpisode>>,
}
impl XtreamSeriesInfoEpisode {
pub fn get_additional_properties(&self, series_info: &XtreamSeriesInfo) -> Option<Box<RawValue>> {
let mut result = serde_json::Map::new();
let info = series_info.info.as_ref();
let bdpath = info.and_then(|i| i.backdrop_path.as_ref());
let bdpath_is_set = bdpath.as_ref().is_some_and(|bdpath| !bdpath.is_empty());
if bdpath_is_set {
result.insert(String::from("backdrop_path"), Value::Array(Vec::from([Value::String(String::from(bdpath?.first()?))])));
}
add_str_property_if_exists!(result, info.map_or("", |i| i.name.as_str()), "series_name");
add_str_property_if_exists!(result, info.map_or("", |i| get_non_empty_str(i.release_date.as_str(), i.releaseDate.as_str(), i.releasedate.as_str())), "series_release_date");
add_str_property_if_exists!(result, self.added.as_str(), "added");
add_str_property_if_exists!(result, info.map_or("", |i| i.cast.as_str()), "cast");
add_str_property_if_exists!(result, self.container_extension.as_str(), "container_extension");
add_str_property_if_exists!(result, self.info.as_ref().map_or("", |info| info.movie_image.as_str()), "cover");
add_str_property_if_exists!(result, info.map_or("", |i| i.director.as_str()), "director");
add_str_property_if_exists!(result, info.map_or("", |i| i.episode_run_time.as_str()), "episode_run_time");
add_str_property_if_exists!(result, info.map_or("", |i| i.last_modified.as_str()), "last_modified");
add_str_property_if_exists!(result, self.info.as_ref().map_or("", |info| info.plot.as_str()), "plot");
add_f64_property_if_exists!(result, info.map_or(0_f64, |i| i.rating), "rating");
add_f64_property_if_exists!(result, info.map_or(0_f64, |i| i.rating_5based), "rating_5based");
add_str_property_if_exists!(result, self.info.as_ref().map_or("", |info| get_non_empty_str(info.release_date.as_str(), info.releaseDate.as_str(), info.releasedate.as_str())), "release_date");
add_str_property_if_exists!(result, self.title, "title");
add_i64_property_if_exists!(result, self.season, "season");
add_i64_property_if_exists!(result, self.episode_num, "episode");
let series_tmdb_id = info.and_then(|i| i.tmdb_id.or(i.tmdb));
add_opt_i64_property_if_exists!(result, self.info.as_ref().and_then(|info| info.tmdb_id.or(info.tmdb.or(series_tmdb_id))), "tmdb_id");
// Add the "info" section to the playlist item additional properties.
if let Some(episode_info) = &self.info {
if let Ok(info_value) = serde_json::to_value(episode_info) {
result.insert("info".to_string(), info_value);
}
}
if result.is_empty() {
None
} else {
let s = serde_json::to_string(&Value::Object(result)).ok()?;
RawValue::from_string(s).ok()
}
}
}
#[allow(clippy::struct_excessive_bools)]
pub struct XtreamMappingOptions {
pub skip_live_direct_source: bool,
pub skip_video_direct_source: bool,
pub skip_series_direct_source: bool,
pub rewrite_resource_url: bool,
pub force_redirect: Option<ClusterFlags>,
}
impl XtreamMappingOptions {
pub fn from_target_options(target: &ConfigTarget, target_output: &XtreamTargetOutput, cfg: &AppConfig) -> Self {
Self {
skip_live_direct_source: target_output.skip_live_direct_source,
skip_video_direct_source: target_output.skip_video_direct_source,
skip_series_direct_source: target_output.skip_series_direct_source,
rewrite_resource_url: cfg.is_reverse_proxy_resource_rewrite_enabled(),
force_redirect: target.options.as_ref().and_then(|o| o.force_redirect),
}
XtreamMappingOptions {
skip_live_direct_source: target_output.skip_live_direct_source,
skip_video_direct_source: target_output.skip_video_direct_source,
skip_series_direct_source: target_output.skip_series_direct_source,
rewrite_resource_url: cfg.is_reverse_proxy_resource_rewrite_enabled(),
force_redirect,
reverse_item_types,
username: user.username.clone(),
password: user.password.clone(),
base_url: resource_url.map(ToString::to_string),
}
}
@@ -596,155 +76,6 @@ pub fn normalize_release_date(document: &mut serde_json::Map<String, Value>) {
}
}
fn append_mandatory_fields(document: &mut serde_json::Map<String, Value>, fields: &[&str]) {
for &field in fields {
if !document.contains_key(field) {
document.insert(field.to_string(), Value::Null);
}
}
}
fn append_prepared_series_properties(add_props: Option<&serde_json::Map<String, Value>>, document: &mut serde_json::Map<String, Value>) {
if let Some(props) = add_props {
match props.get("rating") {
Some(value) => {
document.insert("rating".to_string(), match value {
Value::Number(val) => Value::String(format!("{:.0}", val.as_f64().unwrap_or(0f64))),
Value::String(val) => Value::String(val.clone()),
_ => Value::String("0".to_string()),
});
}
None => {
document.insert("rating".to_string(), Value::String("0".to_string()));
}
}
}
}
fn make_bdpath_resource_url(resource_url: &str, bd_path: &str, index: usize, field_prefix: &str) -> String {
if bd_path.starts_with("http") {
format!("{resource_url}/{field_prefix}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH)
} else {
bd_path.to_string()
}
}
pub fn xtream_playlistitem_to_document(pli: &XtreamPlaylistItem, url: &str, options: &XtreamMappingOptions, user: &ProxyUserCredentials) -> serde_json::Value {
let stream_id_value = Value::Number(serde_json::Number::from(pli.virtual_id));
let is_reverse = user.proxy.is_reverse(pli.item_type) && !options.force_redirect.as_ref().is_some_and(|o| o.has_cluster(pli.item_type));
let (resource_url, logo, logo_small) = if is_reverse && options.rewrite_resource_url {
let resource_url = format!("{url}/resource/{}/{}/{}/{}", pli.xtream_cluster.as_stream_type(), user.username, user.password, pli.get_virtual_id());
let logo_url = if pli.logo.is_empty() { String::new() } else { format!("{resource_url}/logo") };
let logo_small_url = if pli.logo_small.is_empty() { String::new() } else { format!("{resource_url}/logo_small") };
(Some(resource_url), logo_url, logo_small_url)
} else {
(None, pli.logo.clone(), pli.logo_small.clone())
};
let mut document = serde_json::Map::from_iter([
("category_id".to_string(), Value::String(format!("{}", &pli.category_id))),
("category_ids".to_string(), Value::Array(Vec::from([Value::Number(serde_json::Number::from(pli.category_id))]))),
("name".to_string(), Value::String(pli.name.clone())),
("num".to_string(), Value::Number(serde_json::Number::from(pli.channel_no))),
("title".to_string(), Value::String(pli.title.clone())),
("stream_icon".to_string(), Value::String(logo)),
]);
match pli.xtream_cluster {
XtreamCluster::Live => {
document.insert("stream_id".to_string(), stream_id_value);
if options.skip_live_direct_source {
document.insert("direct_source".to_string(), Value::String(String::new()));
} else {
document.insert("direct_source".to_string(), Value::String(pli.url.clone()));
}
document.insert("thumbnail".to_string(), Value::String(logo_small));
document.insert("custom_sid".to_string(), Value::String(String::new()));
document.insert("epg_channel_id".to_string(), pli.epg_channel_id.as_ref().map_or(Value::Null, |epg_id| Value::String(epg_id.clone())));
}
XtreamCluster::Video => {
document.insert("stream_id".to_string(), stream_id_value);
if options.skip_video_direct_source {
document.insert("direct_source".to_string(), Value::String(String::new()));
} else {
document.insert("direct_source".to_string(), Value::String(pli.url.clone()));
}
document.insert("custom_sid".to_string(), Value::String(String::new()));
}
XtreamCluster::Series => {
document.insert("series_id".to_string(), stream_id_value);
}
}
let props = pli.additional_properties.as_ref().and_then(|add_props| serde_json::from_str::<serde_json::Map<String, Value>>(add_props).ok());
if let Some(ref add_props) = props {
let doc = if matches!(pli.xtream_cluster, XtreamCluster::Video | XtreamCluster::Series) {
match add_props.get("info") {
Some(Value::Object(info_doc)) => info_doc,
_ => add_props,
}
} else {
add_props
};
for (field_name, field_value) in doc {
if !document.contains_key(field_name) {
document.insert(field_name.clone(), field_value.to_owned());
}
}
}
match pli.xtream_cluster {
XtreamCluster::Live => {
append_mandatory_fields(&mut document, xtream_const::LIVE_STREAM_FIELDS);
add_to_doc_str_property_if_not_exists!(document, "stream_type", Value::String(String::from("live")));
add_to_doc_str_property_if_not_exists!(document, "added", Value::String(chrono::Utc::now().timestamp().to_string()));
}
XtreamCluster::Video => {
append_mandatory_fields(&mut document, xtream_const::VIDEO_STREAM_FIELDS);
add_to_doc_str_property_if_not_exists!(document, "stream_type", Value::String(String::from("movie")));
add_to_doc_str_property_if_not_exists!(document, "added", Value::String(chrono::Utc::now().timestamp().to_string()));
}
XtreamCluster::Series => {
append_prepared_series_properties(props.as_ref(), &mut document);
append_mandatory_fields(&mut document, xtream_const::SERIES_STREAM_FIELDS);
}
}
rewrite_doc_urls(resource_url.as_ref(), &mut document, xtream_const::XTREAM_VOD_REWRITE_URL_PROPS, "");
Value::Object(document)
}
pub fn rewrite_doc_urls(resource_url: Option<&String>, document: &mut serde_json::Map<String, Value>, fields: &[&str], field_prefix: &str) {
if let Some(rewrite_url) = resource_url {
if let Some(bdpath) = document.get(xtream_const::XC_PROP_BACKDROP_PATH) {
match bdpath {
Value::String(bd_path) => {
document.insert(xtream_const::XC_PROP_BACKDROP_PATH.to_string(), Value::Array(vec![Value::String(make_bdpath_resource_url(rewrite_url.as_str(), bd_path, 0, field_prefix))]));
}
Value::Array(bd_path) => {
document.insert(xtream_const::XC_PROP_BACKDROP_PATH.to_string(), Value::Array(
bd_path.iter()
.filter_map(|val| val.as_str())
.enumerate()
.map(|(index, value)| Value::String(make_bdpath_resource_url(rewrite_url.as_str(), value, index, field_prefix)))
.collect()));
}
_ => {}
}
}
for &field in fields {
if let Some(Value::String(value)) = document.get(field) {
if value.starts_with("http") {
document.insert(field.to_string(), Value::String(format!("{rewrite_url}/{field_prefix}{field}")));
}
}
}
}
}
#[derive(Deserialize, Serialize, Clone)]
pub struct PlaylistXtreamCategory {
#[serde(alias = "category_id")]
backend/src/modules.rs
+1
View File
@@ -13,6 +13,7 @@ macro_rules! include_modules {
pub mod utils;
pub mod tools;
pub mod library;
pub mod ptt;
}
}
backend/src/processing/parser/m3u.rs
+19 -18
View File
@@ -1,9 +1,9 @@
use crate::model::{Config, ConfigInput};
use crate::utils::request::DynReader;
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster, DEFAULT_VIDEO_EXTENSIONS};
use shared::utils::extract_id_from_url;
use std::borrow::BorrowMut;
use tokio::io::AsyncBufReadExt;
use crate::utils::request::DynReader;
// other implementations like calculating text_distance on all titles took too much time
// we keep it now as simple as possible and less memory intensive.
@@ -126,7 +126,7 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
let mut c = skip_digit(&mut it);
loop {
match c {
None=> break,
None => break,
Some(chr) => {
if chr.is_whitespace() {
// skip
@@ -142,18 +142,19 @@ fn process_header(input_name: &str, video_suffixes: &[&str], content: &str, url:
if !value.is_empty() {
provider_id = Some(value);
}
} else if token == "tvg-chno" {
plih.chno = value.parse::<u32>().unwrap_or(0);
} else {
process_header_fields!(plih, token.as_str(),
(id, "tvg-id"),
(group, "group-title"),
(name, "tvg-name"),
(chno, "tvg-chno"),
(parent_code, "parent-code"),
(audio_track, "audio-track"),
(logo, "tvg-logo"),
(logo_small, "tvg-logo-small"),
(time_shift, "timeshift"),
(rec, "tvg-rec"); value);
(id, "tvg-id"),
(group, "group-title"),
(name, "tvg-name"),
(parent_code, "parent-code"),
(audio_track, "audio-track"),
(logo, "tvg-logo"),
(logo_small, "tvg-logo-small"),
(time_shift, "timeshift"),
(rec, "tvg-rec"); value);
}
}
}
@@ -202,10 +203,10 @@ pub async fn consume_m3u<F: FnMut(PlaylistItem)>(cfg: &Config, input: &ConfigInp
let input_name = input.name.as_str();
let video_suffixes = match cfg.video.as_ref() {
Some(config) => {
config.extensions.iter().map(String::as_str).collect::<Vec<&str>>()
},
None => DEFAULT_VIDEO_EXTENSIONS.to_vec()
Some(config) => {
config.extensions.iter().map(String::as_str).collect::<Vec<&str>>()
}
None => DEFAULT_VIDEO_EXTENSIONS.to_vec()
};
let mut lines = tokio::io::BufReader::new(lines).lines();
while let Ok(Some(line)) = lines.next_line().await {
@@ -292,7 +293,7 @@ mod test {
assert_eq!(pli.title, "Seven");
assert_eq!(pli.id, "abc-seven");
assert_eq!(pli.logo, "https://abc.nz/.images/seven.png");
assert_eq!(pli.chno, "7");
assert_eq!(pli.chno, 7);
assert_eq!(pli.group, "Sydney");
}
@@ -307,7 +308,7 @@ mod test {
assert_eq!(pli.title, "Seven");
assert_eq!(pli.id, "abc-seven");
assert_eq!(pli.logo, "https://abc.nz/.images/seven.png");
assert_eq!(pli.chno, "7");
assert_eq!(pli.chno, 7);
assert_eq!(pli.group, "Sydney");
}
backend/src/processing/parser/xtream.rs
+95 -113
View File
@@ -1,14 +1,13 @@
use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind};
use std::borrow::Cow;
use crate::model::ConfigInput;
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
use crate::model::{XtreamCategory, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamStream};
use shared::utils::{generate_playlist_uuid, trim_last_slash};
use crate::utils::xtream::{get_xtream_stream_url_base};
use serde_json::Value;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::task::spawn_blocking;
use crate::model::{XtreamCategory};
use crate::utils::request::DynReader;
use crate::utils::xtream::get_xtream_stream_url_base;
use shared::error::{create_tuliprox_error_result, TuliproxError, TuliproxErrorKind};
use shared::model::{EpisodeStreamProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, XtreamCluster};
use shared::utils::{generate_playlist_uuid, trim_last_slash};
use std::collections::HashMap;
use tokio::task::spawn_blocking;
async fn map_to_xtream_category(categories: DynReader) -> Result<Vec<XtreamCategory>, TuliproxError> {
spawn_blocking(move || {
@@ -22,85 +21,78 @@ async fn map_to_xtream_category(categories: DynReader) -> Result<Vec<XtreamCateg
}).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream categories failed: {e}")))?
}
async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader) -> Result<Vec<XtreamStream>, TuliproxError> {
async fn map_to_xtream_streams(xtream_cluster: XtreamCluster, streams: DynReader) -> Result<Vec<StreamProperties>, TuliproxError> {
spawn_blocking(move || {
let reader = tokio_util::io::SyncIoBridge::new(streams);
match serde_json::from_reader::<_, Vec<XtreamStream>>(reader) {
Ok(stream_list) => Ok(stream_list),
Err(err) => {
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {xtream_cluster}: {err}", )
let reader = tokio_util::io::SyncIoBridge::new(streams);
let parsed: Result<Vec<StreamProperties>, serde_json::Error> = match xtream_cluster {
XtreamCluster::Live => serde_json::from_reader::<_, Vec<LiveStreamProperties>>(reader).map(|list| list.into_iter().map(StreamProperties::Live).collect()),
XtreamCluster::Video => serde_json::from_reader::<_, Vec<VideoStreamProperties>>(reader).map(|list| list.into_iter().map(Box::new).map(StreamProperties::Video).collect()),
XtreamCluster::Series => serde_json::from_reader::<_, Vec<SeriesStreamProperties>>(reader).map(|list| list.into_iter().map(Box::new).map(StreamProperties::Series).collect()),
};
match parsed {
Ok(mut stream_list) => {
for stream in &mut stream_list {
stream.prepare();
}
Ok(stream_list)
}
Err(err) => {
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to map to xtream streams {xtream_cluster}: {err}", )
}
}
}
}).await.map_err(|e| TuliproxError::new(TuliproxErrorKind::Notify, format!("Mapping xtream streams failed: {e}")))?
}
fn create_xtream_series_episode_url(url: &str, username: &str, password: &str, episode: &XtreamSeriesInfoEpisode) -> Arc<String> {
fn create_xtream_series_episode_url<'a>(url: &'a str, username: &'a str, password: &'a str, episode: &'a SeriesStreamDetailEpisodeProperties) -> Cow<'a, str> {
if episode.direct_source.is_empty() {
let ext = episode.container_extension.clone();
let stream_base_url = format!("{url}/series/{username}/{password}/{}.{ext}", episode.id);
Arc::new(stream_base_url)
Cow::Owned(stream_base_url)
} else {
Arc::new(episode.direct_source.clone())
Cow::Borrowed(&episode.direct_source)
}
}
pub fn parse_xtream_series_info(info: &Value, group_title: &str, series_name: &str, input: &ConfigInput) -> Result<Option<Vec<(XtreamSeriesInfoEpisode, PlaylistItem)>>, TuliproxError> {
pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStreamProperties, group_title: &str, series_name: &str, input: &ConfigInput) -> Option<Vec<PlaylistItem>> {
let url = input.url.as_str();
let username = input.username.as_ref().map_or("", |v| v);
let password = input.password.as_ref().map_or("", |v| v);
match serde_json::from_value::<XtreamSeriesInfo>(info.to_owned()) {
Ok(series_info) => {
if let Some(episodes) = &series_info.episodes {
let result: Vec<(XtreamSeriesInfoEpisode, PlaylistItem)> = episodes.iter().map(|episode| {
let episode_url = create_xtream_series_episode_url(url, username, password, episode);
let mut new_episode = episode.clone();
// We need to set the tmdb_id and tmdb from the series info if it is not set.
if let Some(episode_info) = &mut new_episode.info {
if episode_info.tmdb_id.is_none_or(|id| id == 0) && episode_info.tmdb.is_none_or(|id| id == 0) {
let series_tmdb_id = series_info.info.as_ref().and_then(|i| i.tmdb_id.or(i.tmdb));
episode_info.tmdb_id = series_tmdb_id;
episode_info.tmdb = series_tmdb_id;
}
}
(new_episode,
PlaylistItem {
header: PlaylistItemHeader {
id: episode.id.to_string(),
uuid: generate_playlist_uuid(&input.name, &episode.id.to_string(), PlaylistItemType::Series, &episode_url),
name: series_name.to_string(),
logo: episode.info.as_ref().map_or_else(String::new, |info| info.movie_image.clone()),
group: group_title.to_string(),
title: episode.title.clone(),
url: episode_url.to_string(),
item_type: PlaylistItemType::Series,
xtream_cluster: XtreamCluster::Series,
additional_properties: episode.get_additional_properties(&series_info),
category_id: 0,
input_name: input.name.clone(),
..Default::default()
}
})
}).collect();
return if result.is_empty() { Ok(None) } else { Ok(Some(result)) };
}
Ok(None)
}
Err(err) => {
create_tuliprox_error_result!(TuliproxErrorKind::Notify, "Failed to process series info for {series_name} {err}")
}
if let Some(episodes) = series_info.details.as_ref().and_then(|d| d.episodes.as_ref()) {
let result: Vec<PlaylistItem> = episodes.iter().map(|episode| {
let episode_url = create_xtream_series_episode_url(url, username, password, episode);
let episode_info = EpisodeStreamProperties::from_series(series_info, episode);
PlaylistItem {
header: PlaylistItemHeader {
id: episode.id.to_string(),
uuid: generate_playlist_uuid(&input.name, &episode.id.to_string(), PlaylistItemType::Series, &episode_url),
// we use parent_code to track the parent series
parent_code: parent_uuid.to_string(),
name: series_name.to_string(),
logo: episode.movie_image.clone(),
group: group_title.to_string(),
title: episode.title.clone(),
url: episode_url.to_string(),
item_type: PlaylistItemType::Series,
xtream_cluster: XtreamCluster::Series,
additional_properties: Some(StreamProperties::Episode(episode_info)),
category_id: 0,
input_name: input.name.clone(),
..Default::default()
}
}
}).collect();
return if result.is_empty() { None } else { Some(result) };
}
None
}
#[allow(clippy::too_many_arguments)]
pub fn get_xtream_url(xtream_cluster: XtreamCluster, url: &str,
username: &str, password: &str,
stream_id: u32, container_extension: Option<&String>,
stream_id: u32, container_extension: Option<&str>,
live_stream_use_prefix: bool, live_stream_without_extension: bool) -> String {
let url = trim_last_slash(url);
let stream_base_url = match xtream_cluster {
@@ -110,8 +102,11 @@ pub fn get_xtream_url(xtream_cluster: XtreamCluster, url: &str,
format!("{url}/{ctx_path}{username}/{password}/{stream_id}{suffix}")
}
XtreamCluster::Video => {
let ext = container_extension.as_ref().map_or("mp4", |e| e.as_str());
format!("{url}/movie/{username}/{password}/{stream_id}.{ext}")
if let Some(extension) = container_extension {
format!("{url}/movie/{username}/{password}/{stream_id}.{extension}")
} else {
format!("{url}/movie/{username}/{password}/{stream_id}")
}
}
XtreamCluster::Series =>
format!("{}&action={}&series_id={stream_id}", get_xtream_stream_url_base(url.as_ref(), username, password), crate::model::XC_ACTION_GET_SERIES_INFO)
@@ -119,21 +114,19 @@ pub fn get_xtream_url(xtream_cluster: XtreamCluster, url: &str,
stream_base_url
}
pub fn create_xtream_url(xtream_cluster: XtreamCluster, url: &str, username: &str, password: &str,
stream: &XtreamStream, live_stream_use_prefix: bool, live_stream_without_extension: bool) -> String {
if stream.direct_source.is_empty() {
get_xtream_url(xtream_cluster, url, username, password, stream.get_stream_id(),
stream.container_extension.as_ref().map(std::string::ToString::to_string).as_ref(),
live_stream_use_prefix, live_stream_without_extension)
} else {
stream.direct_source.clone()
}
pub fn create_xtream_url<'a>(xtream_cluster: XtreamCluster, url: &'a str, username: &'a str, password: &'a str,
stream: &'a StreamProperties, live_stream_use_prefix: bool, live_stream_without_extension: bool) -> Cow<'a, str> {
stream.get_direct_source().unwrap_or_else(||
Cow::Owned(get_xtream_url(xtream_cluster, url, username, password, stream.get_stream_id(),
stream.get_container_extension().as_deref(),
live_stream_use_prefix, live_stream_without_extension))
)
}
pub async fn parse_xtream(input: &ConfigInput,
xtream_cluster: XtreamCluster,
categories: DynReader,
streams: DynReader) -> Result<Option<Vec<PlaylistGroup>>, TuliproxError> {
xtream_cluster: XtreamCluster,
categories: DynReader,
streams: DynReader) -> Result<Option<Vec<PlaylistGroup>>, TuliproxError> {
match map_to_xtream_category(categories).await {
Ok(xtream_categories) => {
let input_name = input.name.clone();
@@ -142,13 +135,11 @@ pub async fn parse_xtream(input: &ConfigInput,
let password = input.password.as_ref().map_or("", |v| v);
match map_to_xtream_streams(xtream_cluster, streams).await {
Ok(mut xtream_streams) => {
let mut group_map: HashMap<String, XtreamCategory> =
xtream_categories.into_iter().map(|category|
(category.category_id.clone(), category)
).collect();
Ok(xtream_streams) => {
let mut group_map: HashMap<u32, XtreamCategory> =
xtream_categories.into_iter().map(|category| (category.category_id, category)).collect();
let mut unknown_grp = XtreamCategory {
category_id: "0".to_string(),
category_id: 0u32,
category_name: "Unknown".to_string(),
channels: vec![],
};
@@ -156,33 +147,24 @@ pub async fn parse_xtream(input: &ConfigInput,
let (live_stream_use_prefix, live_stream_without_extension) = input.options.as_ref()
.map_or((true, false), |o| (o.xtream_live_stream_use_prefix, o.xtream_live_stream_without_extension));
for stream in &mut xtream_streams {
let group = group_map.get_mut(&stream.category_id).unwrap_or(&mut unknown_grp);
for stream in xtream_streams {
let group = group_map.get_mut(&stream.get_category_id()).unwrap_or(&mut unknown_grp);
let category_name = &group.category_name;
let stream_url = create_xtream_url(xtream_cluster, url, username, password, stream, live_stream_use_prefix, live_stream_without_extension);
let stream_url = create_xtream_url(xtream_cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension);
let item_type = PlaylistItemType::from(xtream_cluster);
// EPG Channel id fix, remove empty
stream.epg_channel_id = if let XtreamCluster::Live = xtream_cluster {
stream.epg_channel_id.as_ref()
.filter(|epg_id| !epg_id.trim().is_empty())
.map(|epg_id| epg_id.to_lowercase())
.or(None)
} else {
None
};
let item = PlaylistItem {
header: PlaylistItemHeader {
id: stream.get_stream_id().to_string(),
uuid: generate_playlist_uuid(&input_name, &stream.get_stream_id().to_string(), item_type, &stream_url),
name: stream.name.clone(),
logo: stream.stream_icon.clone(),
name: stream.get_name().to_string(),
logo: stream.get_stream_icon().to_string(),
group: category_name.clone(),
title: stream.name.clone(),
url: stream_url.clone(),
epg_channel_id: stream.epg_channel_id.clone(),
title: stream.get_name().to_string(),
url: stream_url.to_string(),
epg_channel_id: stream.get_epg_channel_id(),
item_type,
xtream_cluster,
additional_properties: stream.get_additional_properties(),
additional_properties: Some(stream),
category_id: 0,
input_name: input_name.clone(),
..Default::default()
@@ -190,15 +172,17 @@ pub async fn parse_xtream(input: &ConfigInput,
};
group.add(item);
}
let has_channels = !unknown_grp.channels.is_empty();
if has_channels {
group_map.insert("0".to_string(), unknown_grp);
group_map.insert(0, unknown_grp);
}
Ok(Some(group_map.values().filter(|category| !category.channels.is_empty())
.map(|category| {
PlaylistGroup {
id: category.category_id.parse::<u32>().unwrap_or(0),
id: category.category_id,
xtream_cluster,
title: category.category_name.clone(),
channels: category.channels.clone(),
@@ -214,25 +198,23 @@ pub async fn parse_xtream(input: &ConfigInput,
#[cfg(test)]
mod tests {
use std::fs;
use shared::model::XtreamCluster;
use crate::model::{XtreamSeriesInfo};
use crate::processing::parser::xtream::map_to_xtream_streams;
use crate::utils::async_file_reader;
use shared::model::{XtreamCluster, XtreamSeriesInfo};
use std::fs;
#[test]
fn test_read_json_file_into_struct() {
let file_content = fs::read_to_string("/tmp/series-info.json").expect("Unable to read file");
match serde_json::from_str::<XtreamSeriesInfo>(&file_content) {
match serde_json::from_str::<XtreamSeriesInfo>(&file_content) {
Ok(series_info) => {
println!("{:#?}", series_info);
assert!(true);
},
}
Err(err) => {
assert!(false, "Failed to parse json file: {err}");
}
}
}
#[tokio::test]
@@ -244,7 +226,7 @@ mod tests {
println!("{:?}", _streams.get(100));
println!("{:?}", _streams.get(200));
assert!(true);
},
}
Err(err) => {
assert!(false, "Failed to parse json file: {err}");
}
backend/src/processing/playlist_watch.rs
+4 -4
View File
@@ -5,7 +5,7 @@ use shared::model::{MsgKind, PlaylistGroup};
use crate::messaging::{send_message};
use crate::model::Config;
use crate::utils;
use crate::utils::{bincode_deserialize, bincode_serialize};
use crate::utils::{binary_deserialize, binary_serialize};
pub async fn process_group_watch(client: &reqwest::Client, cfg: &Config, target_name: &str, pl: &PlaylistGroup) {
let mut new_tree = BTreeSet::new();
@@ -20,7 +20,7 @@ pub async fn process_group_watch(client: &reqwest::Client, cfg: &Config, target_
Some(path) => {
let save_path = path.as_path();
let mut changed = false;
if path.exists() {
if let Ok(true) = tokio::fs::try_exists(&path).await {
if let Some(loaded_tree) = load_watch_tree(&path).await {
// Find elements in set2 but not in set1
let added_difference: BTreeSet<String> = new_tree.difference(&loaded_tree).cloned().collect();
@@ -79,11 +79,11 @@ async fn handle_watch_notification(client: &reqwest::Client, cfg: &Config, added
async fn load_watch_tree(path: &Path) -> Option<BTreeSet<String>> {
let encoded = tokio::fs::read(path).await.ok()?;
bincode_deserialize(&encoded[..]).ok()
binary_deserialize(&encoded[..]).ok()
}
async fn save_watch_tree(path: &Path, tree: &BTreeSet<String>) -> std::io::Result<()> {
let encoded: Vec<u8> = bincode_serialize(&tree)?;
let encoded: Vec<u8> = binary_serialize(&tree)?;
tokio::fs::write(path, encoded).await
}
backend/src/processing/processor/epg.rs
+2 -2
View File
@@ -64,7 +64,7 @@ impl EpgIdCache<'_> {
/// cache.normalize_and_store("Discovery Channel", Some(&"discovery.epg".to_string()));
/// assert!(cache.normalized.contains_key(&cache.normalize("Discovery Channel")));
/// ```
fn normalize_and_store(&mut self, name: &str, epg_id: Option<&String>) {
fn normalize_and_store(&mut self, name: &str, epg_id: Option<&str>) {
self.insert_normalized(name);
if let Some(chan_epg_id) = epg_id {
@@ -124,7 +124,7 @@ impl EpgIdCache<'_> {
if needs_normalization {
let name = &channel.header.name;
self.normalize_and_store(name, channel.header.epg_channel_id.as_ref());
self.normalize_and_store(name, channel.header.epg_channel_id.as_deref());
}
}
}
backend/src/processing/processor/library.rs
+169 -172
View File
@@ -1,14 +1,12 @@
use crate::library::{MediaMetadata, MetadataAsyncIter, MetadataCacheEntry};
use crate::model::{AppConfig, ConfigInput, XtreamSeriesInfo, XtreamSeriesInfoEpisode, XtreamSeriesInfoEpisodeInfo, XtreamSeriesInfoInfo};
use crate::model::{XtreamMovieData, XtreamMovieInfo, XtreamMovieInfoDetails};
use serde_json::value::RawValue;
use crate::model::{AppConfig, ConfigInput};
use shared::error::TuliproxError;
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster};
use shared::utils::{generate_playlist_uuid, string_to_uuid_type};
use shared::model::{EpisodeStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster};
use shared::utils::{generate_playlist_uuid};
use std::path::Path;
use std::sync::Arc;
pub async fn get_library_playlist(_client: &reqwest::Client, app_config: &Arc<AppConfig>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
pub async fn download_library_playlist(_client: &reqwest::Client, app_config: &Arc<AppConfig>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let config = &*app_config.config.load();
let Some(library_config) = config.library.as_ref() else { return (vec![], vec![]) };
if !library_config.enabled { return (vec![], vec![]); }
@@ -56,16 +54,14 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s
match metadata {
MediaMetadata::Movie(_) => {
let additional_properties = metadata_cache_entry_to_xtream_movie_info(entry).and_then(|info| {
let raw = serde_json::to_string(&info).ok()?;
RawValue::from_string(raw).ok()
});
let additional_properties = metadata_cache_entry_to_xtream_movie_info(entry);
vec![PlaylistItem {
header: PlaylistItemHeader {
uuid: string_to_uuid_type(&entry.uuid),
uuid: UUIDType::from_valid_uuid(&entry.uuid),
name: metadata.title().to_string(),
group: group_name.to_string(),
title: metadata.title().to_string(),
logo: metadata.poster().map_or_else(String::new, ToString::to_string),
url: format!("file://{}", entry.file_path),
xtream_cluster: XtreamCluster::Video,
additional_properties,
@@ -77,48 +73,71 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s
}
MediaMetadata::Series(_series) => {
let mut items = vec![];
if let Some(xtream_series_info) = metadata_cache_entry_to_xtream_series_info(entry) {
let additional_properties = serde_json::to_string(&xtream_series_info).ok().and_then(|r| RawValue::from_string(r).ok());
if let Some(additional_properties) = metadata_cache_entry_to_xtream_series_info(entry) {
let mut episodes = vec![];
if let StreamProperties::Series(series_properties) = &additional_properties {
if let Some(details_props) = series_properties.details.as_ref() {
if let Some(prop_episodes) = details_props.episodes.as_ref() {
for episode in prop_episodes {
let logo = if episode.movie_image.is_empty() { metadata.poster().map_or_else(String::new, ToString::to_string) } else { episode.movie_image.clone() };
let container_extension = Path::new(&episode.direct_source)
.extension()
.and_then(|s| s.to_str())
.map(ToString::to_string).unwrap_or_default();
episodes.push(PlaylistItem {
header: PlaylistItemHeader {
id: episode.id.to_string(),
// we use parent_code for local series to find the parent series info and straighten the virtual_ids
parent_code: entry.uuid.clone(),
uuid: generate_playlist_uuid(input_name, &episode.id.to_string(), PlaylistItemType::LocalSeries, &episode.direct_source),
logo: logo.clone(),
name: episode.title.clone(),
group: group_name.to_string(),
title: episode.title.clone(),
url: episode.direct_source.clone(),
xtream_cluster: XtreamCluster::Series,
item_type: PlaylistItemType::LocalSeries,
category_id: 0,
input_name: input_name.to_string(),
additional_properties: Some(StreamProperties::Episode(EpisodeStreamProperties {
episode_id: episode.id,
episode: episode.episode_num,
season: episode.season,
added: Some(episode.added.clone()),
release_date: Some(episode.release_date.clone()),
tmdb: episode.tmdb,
movie_image: logo,
container_extension,
audio: None,
video: None,
})),
..Default::default()
}
});
}
}
}
}
let series_info = PlaylistItem {
header: PlaylistItemHeader {
uuid: string_to_uuid_type(&entry.uuid),
uuid: UUIDType::from_valid_uuid(&entry.uuid),
id: entry.uuid.clone(),
name: metadata.title().to_string(),
group: group_name.to_string(),
title: metadata.title().to_string(),
logo: metadata.poster().map_or_else(String::new, ToString::to_string),
url: format!("file://{}", entry.file_path),
xtream_cluster: XtreamCluster::Series,
item_type: PlaylistItemType::LocalSeriesInfo,
input_name: input_name.to_string(),
additional_properties,
additional_properties: Some(additional_properties),
..PlaylistItemHeader::default()
}
};
items.push(series_info);
if let Some(episodes) = xtream_series_info.episodes.as_ref() {
for episode in episodes {
items.push(PlaylistItem {
header: PlaylistItemHeader {
id: episode.id.to_string(),
// we use parent_code for local series to find the parent series info and straighten the virtual_ids
parent_code: entry.uuid.clone(),
uuid: generate_playlist_uuid(input_name, &episode.id.to_string(), PlaylistItemType::LocalSeries, &episode.direct_source),
name: episode.title.clone(),
group: group_name.to_string(),
title: episode.title.clone(),
url: episode.direct_source.clone(),
xtream_cluster: XtreamCluster::Series,
item_type: PlaylistItemType::LocalSeries,
category_id: 0,
input_name: input_name.to_string(),
additional_properties: episode.get_additional_properties(&xtream_series_info),
..Default::default()
}
});
}
}
items.extend(episodes);
}
items
@@ -128,7 +147,7 @@ fn to_playlist_item(entry: &MetadataCacheEntry, input_name: &str, group_name: &s
pub fn metadata_cache_entry_to_xtream_movie_info(
entry: &MetadataCacheEntry,
) -> Option<XtreamMovieInfo> {
) -> Option<StreamProperties> {
let movie = match &entry.metadata {
MediaMetadata::Movie(m) => m,
MediaMetadata::Series(_) => return None,
@@ -137,96 +156,123 @@ pub fn metadata_cache_entry_to_xtream_movie_info(
let container_extension = Path::new(&entry.file_path)
.extension()
.and_then(|s| s.to_str())
.map(ToString::to_string);
.map(ToString::to_string).unwrap_or_default();
let actor_names = movie.actors.as_ref().map(|a| a.iter().map(|a| a.name.clone()).collect::<Vec<_>>().join(", "));
let info = XtreamMovieInfoDetails {
kinopoisk_url: movie.tmdb_id.map(|id| format!("https://www.themoviedb.org/movie/{id}")),
tmdb_id: movie.tmdb_id.map(|id| id.to_string()),
name: Some(movie.title.clone()),
o_name: movie.original_title.clone(),
cover_big: movie.poster.clone(),
movie_image: movie.poster.clone(),
releasedate: movie.year.map(|y| format!("{y}-01-01")),
episode_run_time: movie.runtime,
director: movie.directors.as_ref().map(|d| d.join(", ")),
youtube_trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())),
actors: actor_names.clone(),
cast: actor_names,
genre: movie.genres.as_ref().map(|g| g.join(", ")),
description: movie.plot.clone(),
plot: movie.plot.clone(),
age: None,
mpaa_rating: movie.mpaa.clone(),
rating_count_kinopoisk: 0,
country: None,
backdrop_path: movie
.fanart
.as_ref()
.filter(|s| !s.is_empty())
.map(|f| vec![f.clone()])
.or_else(|| {
movie.poster
.as_ref()
.filter(|s| !s.is_empty())
.map(|p| vec![p.clone()])
})
.unwrap_or_default(),
duration_secs: movie.runtime.map(|r| (r * 60).to_string()),
duration: movie.runtime.map(|r| {
let h = r / 60;
let m = r % 60;
format!("{h:02}:{m:02}:00")
}),
video: Vec::new(),
audio: Vec::new(),
bitrate: 0,
rating: movie.rating.map(|r| format!("{r:.2}")),
runtime: movie.runtime.map(|r| (r * 60).to_string()),
status: Some("Released".to_string()),
};
let movie_data = XtreamMovieData {
stream_id: 0,
let properties = VideoStreamProperties {
name: movie.title.clone(),
added: Some(entry.file_modified.to_string()),
category_id: 0,
category_ids: vec![0],
container_extension,
custom_sid: None,
stream_id: 0,
stream_icon: movie.poster.as_deref().or(movie.fanart.as_deref()).unwrap_or("").to_owned(),
direct_source: String::new(),
custom_sid: None,
added: entry.file_modified.to_string(),
container_extension,
rating: movie.rating,
rating_5based: None,
stream_type: Some("movie".to_string()),
trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())),
tmdb: movie.tmdb_id,
is_adult: 0,
details: Some(VideoStreamDetailProperties {
kinopoisk_url: movie.tmdb_id.map(|id| format!("https://www.themoviedb.org/movie/{id}")),
o_name: movie.original_title.clone(),
cover_big: movie.poster.clone(),
movie_image: movie.poster.clone(),
release_date: movie.year.map(|y| format!("{y}-01-01")),
episode_run_time: movie.runtime,
director: movie.directors.as_ref().map(|d| d.join(", ")),
youtube_trailer: movie.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())),
actors: actor_names.clone(),
cast: actor_names,
genre: movie.genres.as_ref().map(|g| g.join(", ")),
description: movie.plot.clone(),
plot: movie.plot.clone(),
age: None,
mpaa_rating: movie.mpaa.clone(),
rating_count_kinopoisk: 0,
country: None,
backdrop_path: movie
.fanart
.as_ref()
.filter(|s| !s.is_empty())
.map(|f| vec![f.clone()])
.or_else(|| {
movie.poster
.as_ref()
.filter(|s| !s.is_empty())
.map(|p| vec![p.clone()])
}),
duration_secs: movie.runtime.map(|r| (r * 60).to_string()),
duration: movie.runtime.map(|r| {
let h = r / 60;
let m = r % 60;
format!("{h:02}:{m:02}:00")
}),
video: None,
audio: None,
bitrate: 0,
runtime: movie.runtime.map(|r| (r * 60).to_string()),
status: Some("Released".to_string()),
}),
};
Some(XtreamMovieInfo { info, movie_data })
Some(StreamProperties::Video(Box::new(properties)))
}
pub fn metadata_cache_entry_to_xtream_series_info(
entry: &MetadataCacheEntry,
) -> Option<XtreamSeriesInfo> {
) -> Option<StreamProperties> {
let series = match &entry.metadata {
MediaMetadata::Movie(_) => return None,
MediaMetadata::Series(m) => m,
};
let actor_names = series.actors.as_ref().map(|a| a.iter().map(|a| a.name.clone()).collect::<Vec<_>>().join(", ")).unwrap_or_default();
let release_date = series.year.map(|y| format!("{y}-01-01")).unwrap_or_default();
let release_date = series.year.map(|y| format!("{y}-01-01"));
let youtube_trailer = series.videos.as_ref().and_then(|v| v.iter().find(|video| video.site.eq_ignore_ascii_case("youtube")).map(|video| video.key.clone())).unwrap_or_default();
let info = XtreamSeriesInfoInfo {
let episodes = series.episodes.as_ref().map(|episodes| {
episodes.iter().filter(|episode| !episode.file_path.is_empty()).map(|episode| {
let container_extension = Path::new(&episode.file_path)
.extension()
.and_then(|s| s.to_str())
.map(ToString::to_string)
.unwrap_or_default();
let episode_release_date = episode.aired.as_ref().map(ToString::to_string).unwrap_or_default();
let tmdb_id = (episode.tmdb_id > 0).then_some(episode.tmdb_id);
SeriesStreamDetailEpisodeProperties {
id: tmdb_id.unwrap_or_default(),
episode_num: episode.episode,
season: episode.season,
title: episode.title.clone(),
container_extension,
custom_sid: None,
added: episode.file_modified.to_string(),
direct_source: episode.file_path.clone(),
tmdb: tmdb_id,
release_date: episode_release_date.clone(),
plot: episode.plot.clone(),
duration_secs: episode.runtime.map_or(0, |r| r * 60),
duration: episode.runtime
.map(|r| format!("{:02}:{:02}:00", r / 60, r % 60))
.unwrap_or_default(),
movie_image: episode.thumb.clone().unwrap_or_default(),
audio: None,
video: None,
bitrate: 0,
rating: None,
}
}).collect::<Vec<_>>()
});
let properties = SeriesStreamProperties {
name: series.title.clone(),
cover: series.poster.clone().unwrap_or_default(),
plot: series.plot.clone().unwrap_or_default(),
cast: actor_names,
director: series.directors.as_ref().map(|d| d.join(", ")).unwrap_or_default(),
genre: series.genres.as_ref().map(|d| d.join(", ")).unwrap_or_default(),
release_date: release_date.clone(),
releaseDate: release_date.clone(),
releasedate: release_date.clone(),
last_modified: series.last_updated.to_string(),
rating: series.rating.unwrap_or(0f64),
rating_5based: 0.0,
category_id: 0,
series_id: 0,
backdrop_path: series
.fanart
.as_ref()
@@ -237,73 +283,24 @@ pub fn metadata_cache_entry_to_xtream_series_info(
.filter(|s| !s.is_empty())
.map(|p| vec![p.clone()])
}),
trailer: youtube_trailer.clone(),
cast: actor_names,
cover: series.poster.clone().unwrap_or_default(),
director: series.directors.as_ref().map(|d| d.join(", ")).unwrap_or_default(),
episode_run_time: None,
genre: series.genres.as_ref().map(|d| d.join(", ")),
last_modified: Some(series.last_updated.to_string()),
plot: series.plot.clone(),
rating: series.rating.unwrap_or(0f64),
rating_5based: 0.0,
release_date,
youtube_trailer,
episode_run_time: String::from("0"),
category_id: 0,
tmdb_id: series.tmdb_id,
tmdb: series.tmdb_id,
year: series.year,
details: Some(SeriesStreamDetailProperties {
year: series.year,
// seasons are not delivered through xtream get_series_info.
episodes,
}),
};
// seasons are not delivered through xtream get_series_info.
let seasons = Some(vec![]); /*series.seasons.as_ref()
.map(|seasons| seasons.iter().map(|season| XtreamSeriesInfoSeason {
air_date: season.air_date.clone().unwrap_or_default(),
episode_count: season.episode_count,
id: season.id,
name: season.name.clone(),
overview: season.overview.clone().unwrap_or_default(),
season_number: season.season_number,
vote_average: season.vote_average,
cover: season.poster_path.as_ref().cloned().unwrap_or_default(),
cover_big: season.poster_path.as_ref().cloned().unwrap_or_default(),
}).collect());
*/
let episodes = series.episodes.as_ref().map(|episodes| {
episodes.iter().filter(|episode| !episode.file_path.is_empty()).map(|episode| {
let container_extension = Path::new(&episode.file_path)
.extension()
.and_then(|s| s.to_str())
.map(ToString::to_string)
.unwrap_or_default();
let episode_release_date = episode.aired.as_ref().map(ToString::to_string).unwrap_or_default();
let tmdb_id = (episode.tmdb_id > 0).then_some(episode.tmdb_id);
XtreamSeriesInfoEpisode {
id: episode.id,
episode_num: episode.episode,
season: episode.season,
title: episode.title.clone(),
container_extension,
info: Some(XtreamSeriesInfoEpisodeInfo {
id: tmdb_id,
tmdb_id,
tmdb: tmdb_id,
season: episode.season,
release_date: episode_release_date.clone(),
releaseDate: episode_release_date.clone(),
releasedate: episode_release_date,
plot: episode.plot.clone().unwrap_or_default(),
duration_secs: episode.runtime.map_or(0, |r| r * 60),
duration: episode.runtime
.map(|r| format!("{:02}:{:02}:00", r / 60, r % 60))
.unwrap_or_default(),
movie_image: episode.thumb.clone().unwrap_or_default(),
video: None,
audio: None,
bitrate: 0,
rating: episode.rating.unwrap_or(0.0),
}),
custom_sid: String::new(),
added: episode.file_modified.to_string(),
direct_source: episode.file_path.clone(),
}
}).collect::<Vec<_>>()
});
Some(XtreamSeriesInfo { seasons, info: Some(info), episodes })
Some(StreamProperties::Series(Box::new(properties)))
}
backend/src/processing/processor/mod.rs
-22
View File
@@ -8,28 +8,6 @@ mod sort;
pub mod trakt;
mod library;
#[macro_export]
macro_rules! handle_error {
($stmt:expr, $map_err:expr) => {
if let Err(err) = $stmt {
$map_err(err);
}
};
}
use handle_error;
#[macro_export]
macro_rules! handle_error_and_return {
($stmt:expr, $map_err:expr) => {
if let Err(err) = $stmt {
$map_err(err);
return Default::default();
}
};
}
use handle_error_and_return;
//
// fn get_resolve_<cluster>_options(target: &ConfigTarget, fpl: &FetchedPlaylist) -> (bool, u16)
//
backend/src/processing/processor/playlist.rs
+26 -16
View File
@@ -1,8 +1,7 @@
use crate::model::{AppConfig, ConfigFavourites, ConfigInput, ConfigRename};
use crate::model::{AppConfig, Config, ConfigFavourites, ConfigInput, ConfigRename};
use crate::utils::epg;
use crate::utils::m3u;
use crate::utils::xtream;
use crate::Config;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
@@ -19,11 +18,12 @@ use crate::model::{InputStats, PlaylistStats, SourceStats, TargetStats};
use crate::processing::parser::xmltv::flatten_tvguide;
use crate::processing::playlist_watch::process_group_watch;
use crate::processing::processor::epg::process_playlist_epg;
use crate::processing::processor::library;
use crate::processing::processor::sort::sort_playlist;
use crate::processing::processor::trakt::process_trakt_categories_for_target;
use crate::processing::processor::xtream_series::playlist_resolve_series;
use crate::processing::processor::xtream_vod::playlist_resolve_vod;
use crate::repository::playlist_repository::persist_playlist;
use crate::repository::playlist_repository::{persist_input_playlist, persist_playlist};
use crate::utils::StepMeasure;
use crate::utils::{debug_if_enabled, trace_if_enabled};
use deunicode::deunicode;
@@ -34,7 +34,6 @@ use shared::foundation::filter::{get_field_value, set_field_value, Filter, Value
use shared::model::{CounterModifier, FieldGetAccessor, FieldSetAccessor, InputType, ItemField, MsgKind, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistUpdateState, ProcessingOrder, UUIDType, XtreamCluster};
use shared::utils::{default_as_default, hash_bytes};
use std::time::Instant;
use crate::processing::processor::library;
fn is_valid(pli: &PlaylistItem, filter: &Filter) -> bool {
let provider = ValueProvider { pli };
@@ -111,17 +110,17 @@ fn filter_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt
fn assign_channel_no_playlist(new_playlist: &mut [PlaylistGroup]) {
let assigned_chnos: HashSet<u32> = new_playlist.iter().flat_map(|g| &g.channels)
.filter(|c| !c.header.chno.is_empty())
.map(|c| c.header.chno.as_str())
.flat_map(str::parse::<u32>).collect();
.filter(|c| c.header.chno != 0)
.map(|c| c.header.chno)
.collect();
let mut chno = 1;
for group in new_playlist {
for chan in &mut group.channels {
if chan.header.chno.is_empty() {
if chan.header.chno == 0 {
while assigned_chnos.contains(&chno) {
chno += 1;
}
chan.header.chno = chno.to_string();
chan.header.chno = chno;
chno += 1;
}
}
@@ -173,7 +172,7 @@ fn rename_playlist(playlist: &mut [PlaylistGroup], target: &ConfigTarget) -> Opt
fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType {
let mut data = Vec::with_capacity(base_uuid.len() + mapping_id.len());
data.extend_from_slice(base_uuid);
data.extend_from_slice(base_uuid.as_ref());
data.extend_from_slice(mapping_id.as_bytes());
hash_bytes(&data)
}
@@ -309,10 +308,10 @@ fn is_target_enabled(target: &ConfigTarget, user_targets: &ProcessTargets) -> bo
async fn playlist_download_from_input(client: &reqwest::Client, app_config: &Arc<AppConfig>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let config = &*app_config.config.load();
match input.input_type {
InputType::M3u => m3u::get_m3u_playlist(client, config, input).await,
InputType::Xtream => xtream::get_xtream_playlist(config, client, input).await,
InputType::M3u => m3u::download_m3u_playlist(client, config, input).await,
InputType::Xtream => xtream::download_xtream_playlist(config, client, input).await,
InputType::M3uBatch | InputType::XtreamBatch => (vec![], vec![]),
InputType::Library => library::get_library_playlist(client, app_config, input).await,
InputType::Library => library::download_library_playlist(client, app_config, input).await,
}
}
@@ -332,13 +331,25 @@ async fn process_source(client: &reqwest::Client, app_config: Arc<AppConfig>, so
if is_input_enabled(input, &user_targets) {
source_downloaded = true;
let start_time = Instant::now();
let (mut playlistgroups, mut error_list) = playlist_download_from_input(client, &app_config, input).await;
// Download playlist for input
let (playlistgroups, mut error_list) = {
let (downloaded_playlist, mut download_err) = playlist_download_from_input(client, &app_config, input).await;
let (playlist, error) = persist_input_playlist(&app_config, input, downloaded_playlist).await;
if let Some(err) = error {
error!("Failed to persist input playlist {}", input.name);
download_err.push(err);
}
(playlist, download_err)
};
// Download epg for input
let (tvguide, mut tvguide_errors) = if error_list.is_empty() {
let working_dir = &app_config.config.load().working_dir;
epg::get_xmltv(client, input, working_dir).await
} else {
(None, vec![])
};
errors.append(&mut error_list);
errors.append(&mut tvguide_errors);
let group_count = playlistgroups.len();
@@ -348,7 +359,6 @@ async fn process_source(client: &reqwest::Client, app_config: Arc<AppConfig>, so
info!("Source is empty {input_name}");
errors.push(notify_err!(format!("Source is empty {input_name}")));
} else {
playlistgroups.iter_mut().for_each(PlaylistGroup::on_load);
source_playlists.push(
FetchedPlaylist {
input,
@@ -533,7 +543,7 @@ fn flatten_groups(playlistgroups: Vec<PlaylistGroup>) -> Vec<PlaylistGroup> {
}
#[allow(clippy::too_many_arguments)]
async fn process_playlist_for_target(app_config: &AppConfig,
async fn process_playlist_for_target(app_config: &Arc<AppConfig>,
client: &reqwest::Client,
playlists: &mut [FetchedPlaylist<'_>],
target: &ConfigTarget,
backend/src/processing/processor/trakt.rs
+6 -27
View File
@@ -1,13 +1,12 @@
use std::borrow::Cow;
use crate::model::{ConfigTarget, TraktListItem, TraktMatchItem};
use shared::model::{FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, TraktContentType, XtreamCluster};
use crate::model::{TraktConfig, TraktListConfig, TraktMatchResult};
use shared::error::TuliproxError;
use crate::utils::{TraktClient, extract_year_from_title, normalize_title_for_matching};
use shared::utils::{get_u32_from_serde_value};
use shared::utils::{CONSTANTS};
use crate::utils::{extract_year_from_title, normalize_title_for_matching, TraktClient};
use crate::utils::{trace_if_enabled, with};
use log::{debug, info, trace, warn};
use shared::error::TuliproxError;
use shared::model::{FieldGetAccessor, FieldSetAccessor, PlaylistGroup, PlaylistItem, TraktContentType, XtreamCluster};
use shared::utils::CONSTANTS;
use std::borrow::Cow;
use strsim::normalized_levenshtein;
fn extract_quality(value: &str) -> Option<&str> {
@@ -37,26 +36,6 @@ fn is_compatible_content_type(cluster: XtreamCluster, content_type: TraktContent
}
}
/// Extract TMDB ID from playlist item
fn extract_tmdb_id_from_playlist_item(item: &PlaylistItem) -> Option<u32> {
if let Some(additional_props) = &item.header.additional_properties {
let props_str = additional_props.get();
if let Ok(props) = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(props_str) {
return props
.get("tmdb_id")
.and_then(get_u32_from_serde_value)
.filter(|&id| id != 0)
.or_else(|| {
props.get("info")
.and_then(|info| info.get("tmdb_id"))
.and_then(get_u32_from_serde_value)
.filter(|&id| id != 0)
});
}
}
None
}
fn calculate_year_bonus(playlist_year: Option<u32>, trakt_year: Option<u32>) -> f64 {
if let (Some(p_year), Some(t_year)) = (playlist_year, trakt_year) {
if p_year == t_year {
@@ -230,7 +209,7 @@ fn match_trakt_items_with_playlist<'a>(
if is_compatible_content_type(channel.header.xtream_cluster, list_config.content_type) {
let normalized_title = normalize_title_for_matching(&channel.header.title);
let channel_year = extract_year_from_title(&channel.header.title);
let channel_tmdb_id = extract_tmdb_id_from_playlist_item(channel);
let channel_tmdb_id = channel.get_tmdb_id();
if let Some(matched) = find_best_match_for_item((channel, normalized_title, channel_year, channel_tmdb_id), &trakt_match_items, list_config) {
matches.push(matched);
}
backend/src/processing/processor/xtream.rs
+4 -98
View File
@@ -1,19 +1,8 @@
use shared::error::{info_err, notify_err};
use shared::error::{str_to_io_error, TuliproxError};
use crate::model::{AppConfig, Config, ConfigInput, InputSource};
use crate::model::{FetchedPlaylist};
use shared::model::{PlaylistEntry, PlaylistItem, PlaylistItemType, XtreamCluster};
use crate::model::normalize_release_date;
use crate::repository::storage::get_input_storage_path;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use crate::repository::bplustree::BPlusTree;
use crate::repository::storage_const;
use crate::repository::xtream_repository::xtream_get_record_file_path;
use crate::utils;
use shared::error::{info_err};
use shared::error::{TuliproxError};
use crate::model::{ConfigInput, InputSource};
use shared::model::{PlaylistEntry, PlaylistItem, XtreamCluster};
use crate::utils::xtream;
use serde_json::{from_str, to_string, Value};
pub(in crate::processing) async fn playlist_resolve_download_playlist_item(client: &reqwest::Client, pli: &PlaylistItem, input: &ConfigInput, errors: &mut Vec<TuliproxError>, resolve_delay: u16, cluster: XtreamCluster) -> Option<String> {
let mut result = None;
@@ -33,86 +22,3 @@ pub(in crate::processing) async fn playlist_resolve_download_playlist_item(clien
}
result
}
pub(in crate::processing) fn normalize_json_content(content: String) -> String {
match from_str::<Value>(&content) {
Ok(mut json_value) => {
if let Some(info) = json_value.get_mut("info").and_then(Value::as_object_mut) {
normalize_release_date(info);
}
to_string(&json_value).unwrap_or(content)
},
Err(_) => content,
}
}
pub(in crate::processing) async fn create_resolve_episode_wal_files(cfg: &Config, input: &ConfigInput) -> Option<(tokio::fs::File, PathBuf)> {
match get_input_storage_path(&input.name, &cfg.working_dir) {
Ok(storage_path) => {
let info_path = storage_path.join(format!("{}.{}", crate::model::XC_FILE_SERIES_EPISODE_RECORD, storage_const::FILE_SUFFIX_WAL));
let info_file = utils::async_append_or_create_file(&info_path).await.ok()?;
Some((info_file, info_path))
}
Err(_) => None
}
}
pub(in crate::processing) async fn create_resolve_info_wal_files(cfg: &Config, input: &ConfigInput, cluster: XtreamCluster) -> Option<(tokio::fs::File, tokio::fs::File, PathBuf, PathBuf)> {
match get_input_storage_path(&input.name, &cfg.working_dir) {
Ok(storage_path) => {
if let Some(file_prefix) = match cluster {
XtreamCluster::Live => None,
XtreamCluster::Video => Some(crate::model::XC_FILE_VOD_INFO),
XtreamCluster::Series => Some(crate::model::XC_FILE_SERIES_INFO)
} {
let content_path = storage_path.join(format!("{file_prefix}_content.{}", storage_const::FILE_SUFFIX_WAL));
let info_path = storage_path.join(format!("{file_prefix}_record.{}", storage_const::FILE_SUFFIX_WAL));
let content_file = utils::async_append_or_create_file(&content_path).await.ok()?;
let info_file = utils::async_append_or_create_file(&info_path).await.ok()?;
return Some((content_file, info_file, content_path, info_path));
}
None
}
Err(_) => None
}
}
pub(in crate::processing) fn should_update_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap<u32, u64>, field: &str) -> (bool, u32, u64) {
let Some(provider_id) = pli.header.get_provider_id() else { return (false, 0, 0) };
let last_modified = pli.header.get_additional_property_as_u64(field);
let old_timestamp = processed_provider_ids.get(&provider_id);
(old_timestamp.is_none()
|| last_modified.is_none()
|| *old_timestamp.unwrap() != last_modified.unwrap(), provider_id, last_modified.unwrap_or(0))
}
pub(in crate::processing) async fn read_processed_info_ids<V, F>(cfg: &AppConfig, errors: &mut Vec<TuliproxError>, fpl: &FetchedPlaylist<'_>,
item_type: PlaylistItemType, extract_ts: F) -> HashMap<u32, u64>
where
F: Fn(&V) -> u64,
V: Serialize + for<'de> Deserialize<'de> + Clone,
{
let mut processed_info_ids = HashMap::new();
let fpl_name = &fpl.input.name;
let file_path = match get_input_storage_path(fpl_name, &cfg.config.load().working_dir)
.map(|storage_path| xtream_get_record_file_path(&storage_path, item_type)).and_then(|opt| opt.ok_or_else(|| str_to_io_error("Not supported")))
{
Ok(file_path) => file_path,
Err(err) => {
errors.push(notify_err!(format!("Could not create storage path for input {fpl_name}: {err}")));
return processed_info_ids;
}
};
{
let file_lock = cfg.file_locks.read_lock(&file_path).await;
if let Ok(info_records) = BPlusTree::<u32, V>::load(&file_path) {
info_records.iter().for_each(|(provider_id, record)| {
processed_info_ids.insert(*provider_id, extract_ts(record));
});
}
drop(file_lock);
}
processed_info_ids
}
backend/src/processing/processor/xtream_series.rs
+55 -211
View File
@@ -1,230 +1,83 @@
use crate::model::{AppConfig, ConfigTarget};
use crate::model::FetchedPlaylist;
use crate::model::{XtreamSeriesEpisode, XtreamSeriesInfoEpisode};
use crate::model::{AppConfig, ConfigTarget};
use crate::processing::parser::xtream::parse_xtream_series_info;
use crate::processing::processor::playlist::ProcessingPipe;
use crate::processing::processor::xtream::normalize_json_content;
use crate::processing::processor::xtream::{create_resolve_episode_wal_files, create_resolve_info_wal_files, playlist_resolve_download_playlist_item, read_processed_info_ids, should_update_info};
use crate::processing::processor::{create_resolve_options_function_for_xtream_target, handle_error, handle_error_and_return};
use crate::processing::processor::xtream::playlist_resolve_download_playlist_item;
use crate::processing::processor::create_resolve_options_function_for_xtream_target;
use crate::repository::storage::get_input_storage_path;
use crate::repository::xtream_repository::{write_series_info_to_wal_file, xtream_get_info_file_paths,
xtream_update_input_info_file, xtream_update_input_series_episodes_record_from_wal_file,
xtream_update_input_series_record_from_wal_file};
use crate::repository::IndexedDocumentReader;
use crate::utils;
use crate::utils::{bincode_serialize, IO_BUFFER_SIZE};
use log::{error, info, log_enabled, warn, Level};
use shared::error::{info_err, notify_err};
use crate::repository::xtream_repository::persists_input_series_info;
use log::{error, info, log_enabled, Level};
use shared::error::TuliproxError;
use shared::model::InputType;
use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemType, XtreamCluster};
use std::collections::{HashMap, HashSet};
use shared::model::{InputType, PlaylistEntry, SeriesStreamProperties, StreamProperties, XtreamSeriesInfo};
use shared::model::{PlaylistGroup, PlaylistItemType, XtreamCluster};
use std::sync::Arc;
use std::time::Instant;
use tokio::io::AsyncWriteExt;
create_resolve_options_function_for_xtream_target!(series);
async fn read_processed_series_info_ids(cfg: &AppConfig, errors: &mut Vec<TuliproxError>, fpl: &FetchedPlaylist<'_>) -> HashMap<u32, u64> {
read_processed_info_ids(cfg, errors, fpl, PlaylistItemType::SeriesInfo, |ts: &u64| *ts).await
}
async fn write_series_episode_record_to_wal_file(
writer: &mut tokio::io::BufWriter<&mut tokio::fs::File>,
provider_id: u32,
episode: &XtreamSeriesInfoEpisode,
) -> std::io::Result<usize> {
let series_episode = XtreamSeriesEpisode::from(episode);
if let Ok(content_bytes) = bincode_serialize(&series_episode) {
writer.write_all(&provider_id.to_le_bytes()).await?;
let content_len = content_bytes.len();
if let Ok(len) = u32::try_from(content_len) {
writer.write_all(&len.to_le_bytes()).await?;
writer.write_all(&content_bytes).await?;
return Ok(content_len + 4usize);
async fn playlist_resolve_series_info(app_config: &Arc<AppConfig>, client: &reqwest::Client, errors: &mut Vec<TuliproxError>,
fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> Vec<PlaylistGroup> {
let input = fpl.input;
let working_dir = &app_config.config.load().working_dir;
let storage_path = match get_input_storage_path(&input.name, working_dir) {
Ok(storage_path) => storage_path,
Err(err) => {
error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name);
return vec![];
}
error!("Cant write to WAL file, content length exceeds u32");
}
Ok(0)
}
fn should_update_series_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap<u32, u64>) -> (bool, u32, u64) {
should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_SERIES_INFO_LAST_MODIFIED)
}
async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, errors: &mut Vec<TuliproxError>,
fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool {
// TODO read existing WAL File and import it to avoid duplicate requests
let mut processed_info_ids: HashMap<u32, u64> = read_processed_series_info_ids(cfg, errors, fpl).await;
let mut fetched_in_run: HashSet<u32> = HashSet::new();
// we can't write to the indexed-document directly because of the write lock and time-consuming operation.
// All readers would be waiting for the lock and the app would be unresponsive.
// We collect the content into a wal file and write it once we collected everything.
let Some((mut wal_content_file, mut wal_record_file, wal_content_path, wal_record_path))
= create_resolve_info_wal_files(&cfg.config.load(), fpl.input, XtreamCluster::Series).await
else { return !processed_info_ids.is_empty(); };
let mut content_writer = utils::async_file_writer(&mut wal_content_file);
let mut record_writer = utils::async_file_writer(&mut wal_record_file);
let mut content_updated = false;
};
let series_info_count = fpl.playlistgroups.iter()
.filter(|&plg| plg.xtream_cluster == XtreamCluster::Series)
.flat_map(|plg| &plg.channels)
.filter(|&pli| pli.header.item_type == PlaylistItemType::SeriesInfo).count();
.filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Series
&& pli.header.item_type == PlaylistItemType::SeriesInfo
&& !pli.has_details()).count();
info!("Found {series_info_count} series info to resolve");
let mut last_log_time = Instant::now();
let mut processed_series_info_count = 0;
let mut write_counter = 0usize;
// LocalSeriesInfo entries are not resolved!
let input = fpl.input;
let mut result: Vec<PlaylistGroup> = vec![];
for plg in &mut fpl.playlistgroups {
if plg.xtream_cluster != XtreamCluster::Series {
continue;
}
for pli in &mut plg.channels {
if pli.header.item_type != PlaylistItemType::SeriesInfo {
continue;
}
let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids);
if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await {
let normalized_content = normalize_json_content(content);
let normalized_str = normalized_content.as_str();
handle_error_and_return!(write_series_info_to_wal_file(provider_id, ts, normalized_str, &mut content_writer, &mut record_writer).await,
|err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}"))));
processed_info_ids.insert(provider_id, ts);
content_updated = true;
write_counter += normalized_str.len();
// periodic flush to bound BufWriter memory
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = content_writer.flush().await {
errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}")));
}
if let Err(err) = record_writer.flush().await {
errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}")));
}
}
}
}
if log_enabled!(Level::Info) {
processed_series_info_count += 1;
if last_log_time.elapsed().as_secs() >= 30 {
info!("resolved {processed_series_info_count}/{series_info_count} series info");
last_log_time = Instant::now();
}
}
}
}
info!("resolved {processed_series_info_count}/{series_info_count} series info");
// content_wal contains the provider_id and series_info with episode listing
// record_wal contains provider_id and timestamp
if content_updated {
handle_error!(content_writer.flush().await,
|err| errors.push(notify_err!(format!("Failed to resolve series, could not write to wal file {err}"))));
handle_error!(record_writer.flush().await,
|err| errors.push(notify_err!(format!("Failed to resolve series tmdb, could not write to wal file {err}"))));
handle_error!(content_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}"))));
handle_error!(record_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info record to wal file {err}"))));
drop(content_writer);
drop(record_writer);
drop(wal_content_file);
drop(wal_record_file);
handle_error!(xtream_update_input_info_file(cfg, fpl.input, &wal_content_path, XtreamCluster::Series).await,
|err| errors.push(err));
handle_error!(xtream_update_input_series_record_from_wal_file(cfg, fpl.input, &wal_record_path).await,
|err| errors.push(err));
}
// we updated now
// - series_info.db which contains the original series_info json
// - series_record.db which contains the series_info provider_id and timestamp
!processed_info_ids.is_empty()
}
async fn process_series_info(
app_config: &AppConfig,
fpl: &mut FetchedPlaylist<'_>,
errors: &mut Vec<TuliproxError>,
) -> Vec<PlaylistGroup> {
let mut result: Vec<PlaylistGroup> = vec![];
let input = fpl.input;
let config = app_config.config.load();
let Ok(Some((info_path, idx_path))) = get_input_storage_path(&input.name, &config.working_dir)
.map(|storage_path| xtream_get_info_file_paths(&storage_path, XtreamCluster::Series))
else {
errors.push(notify_err!("Failed to open input info file for series".to_string()));
return result;
};
let mut write_counter = 0usize;
let _file_lock = app_config.file_locks.read_lock(&info_path).await;
// Contains the Series Info with episode listing
let Ok(mut info_reader) = IndexedDocumentReader::<u32, String>::new(&info_path, &idx_path) else { return result; };
let Some((mut wal_file, wal_path)) = create_resolve_episode_wal_files(&config, input).await else {
errors.push(notify_err!("Could not create wal file for series episodes record".to_string()));
return result;
};
let mut wal_writer = utils::async_file_writer(&mut wal_file);
for plg in fpl
.playlistgroups
.iter_mut()
.filter(|plg| plg.xtream_cluster == XtreamCluster::Series)
{
let mut group_series = vec![];
// Resolve does not handle LocalSeriesInfo
for pli in plg
.channels
.iter_mut()
.filter(|pli| pli.header.item_type == PlaylistItemType::SeriesInfo)
{
let Some(provider_id) = pli.header.get_provider_id() else { continue; };
let Ok(content) = info_reader.get(&provider_id) else { continue; };
if content.is_empty() {
warn!("Series info content is empty, skipping series with provider id: {provider_id}");
for pli in &mut plg.channels {
if pli.header.xtream_cluster != XtreamCluster::Series
|| pli.header.item_type != PlaylistItemType::SeriesInfo
|| pli.has_details() {
continue;
}
match serde_json::from_str::<serde_json::Value>(&content) {
Ok(series_content) => {
let (group, series_name) = {
let header = &pli.header;
(header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() })
};
match parse_xtream_series_info(&series_content, &group, &series_name, input) {
Ok(Some(mut series)) => {
for (episode, pli_episode) in &mut series {
let Some(provider_id) = &pli_episode.header.get_provider_id() else { continue; };
match write_series_episode_record_to_wal_file(&mut wal_writer, *provider_id, episode).await {
Ok(written_bytes) => {
write_counter += written_bytes;
// periodic flush to bound BufWriter memory
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = wal_writer.flush().await {
errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}")));
}
}
}
Err(err) => { errors.push(info_err!(format!("Failed to write to series episode wal file: {err}"))) }
}
let Some(provider_id) = pli.get_provider_id() else { continue; };
processed_series_info_count += 1;
let (group, series_name) = {
let header = &pli.header;
(header.group.clone(), if header.name.is_empty() { header.title.clone() } else { header.name.clone() })
};
if provider_id != 0 {
if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await {
match serde_json::from_str::<XtreamSeriesInfo>(&content) {
Ok(info) => {
let series_stream_props = SeriesStreamProperties::from_info(&info, pli);
// the input db needs to be updated
let _ = persists_input_series_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &series_stream_props).await;
// extract episodes from info
if let Some(episodes) = parse_xtream_series_info(&pli.get_uuid(), &series_stream_props, &group, &series_name, input) {
group_series.extend(episodes.into_iter());
}
group_series.extend(series.into_iter().map(|(_, pli)| pli));
// Update in-memory playlist items with the newly fetched vod info.
// This makes the data available for later processing steps like STRM export.
pli.header.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props)));
}
Ok(None) => {}
Err(err) => {
errors.push(err);
error!("Failed to parse series info for provider_id {provider_id}: {err}");
}
}
}
Err(err) => errors.push(info_err!(format!("Failed to parse JSON: {err}"))),
}
if log_enabled!(Level::Info)
&& last_log_time.elapsed().as_secs() >= 30 {
info!("resolved {processed_series_info_count}/{series_info_count} series info");
last_log_time = Instant::now();
}
}
if !group_series.is_empty() {
@@ -236,19 +89,11 @@ async fn process_series_info(
});
}
}
handle_error!(wal_writer.flush().await, |err| errors.push(notify_err!(format!("Failed to resolve series episodes, could not write to wal file {err}"))));
handle_error!(wal_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync series info to wal file {err}"))));
drop(wal_writer);
drop(wal_file);
handle_error!(xtream_update_input_series_episodes_record_from_wal_file(app_config, input, &wal_path).await,
|err| errors.push(err));
info!("resolved {processed_series_info_count}/{series_info_count} series info");
result
}
pub async fn playlist_resolve_series(cfg: &AppConfig,
pub async fn playlist_resolve_series(cfg: &Arc<AppConfig>,
client: &reqwest::Client,
target: &ConfigTarget,
errors: &mut Vec<TuliproxError>,
@@ -259,8 +104,7 @@ pub async fn playlist_resolve_series(cfg: &AppConfig,
let (resolve_series, resolve_delay) = get_resolve_series_options(target, processed_fpl);
if !resolve_series { return; }
if !playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_delay).await { return; }
let series_playlist = process_series_info(cfg, provider_fpl, errors).await;
let series_playlist = playlist_resolve_series_info(cfg, client, errors, processed_fpl, resolve_delay).await;
if series_playlist.is_empty() { return; }
// original content saved into original list
for plg in &series_playlist {
backend/src/processing/processor/xtream_vod.rs
+42 -142
View File
@@ -1,175 +1,75 @@
use crate::model::{AppConfig, ConfigTarget};
use crate::model::FetchedPlaylist;
use crate::processing::processor::xtream::normalize_json_content;
use crate::processing::processor::xtream::{create_resolve_info_wal_files, playlist_resolve_download_playlist_item, read_processed_info_ids, should_update_info};
use crate::processing::processor::{create_resolve_options_function_for_xtream_target, handle_error, handle_error_and_return};
use crate::repository::xtream_repository::{write_vod_info_to_wal_file, xtream_update_input_info_file, xtream_update_input_vod_record_from_wal_file, InputVodInfoRecord};
use crate::utils;
use crate::utils::IO_BUFFER_SIZE;
use log::{info, log_enabled, Level};
use serde_json::value::RawValue;
use serde_json::{Map, Value};
use shared::error::notify_err;
use crate::model::{AppConfig, ConfigTarget};
use crate::processing::processor::create_resolve_options_function_for_xtream_target;
use crate::processing::processor::xtream::playlist_resolve_download_playlist_item;
use crate::repository::storage::get_input_storage_path;
use crate::repository::xtream_repository::persist_input_vod_info;
use log::{error, info, log_enabled, Level};
use shared::error::TuliproxError;
use shared::model::{InputType, PlaylistEntry};
use shared::model::{PlaylistItem, PlaylistItemType, XtreamCluster};
use shared::utils::{get_string_from_serde_value, get_u32_from_serde_value, get_u64_from_serde_value};
use std::collections::{HashMap, HashSet};
use shared::model::{InputType, PlaylistEntry, StreamProperties, VideoStreamProperties, XtreamVideoInfo};
use shared::model::{PlaylistItemType, XtreamCluster};
use std::sync::Arc;
use std::time::Instant;
use tokio::io::AsyncWriteExt;
create_resolve_options_function_for_xtream_target!(vod);
async fn read_processed_vod_info_ids(cfg: &AppConfig, errors: &mut Vec<TuliproxError>, fpl: &FetchedPlaylist<'_>) -> HashMap<u32, u64> {
read_processed_info_ids(cfg, errors, fpl, PlaylistItemType::Video, |record: &InputVodInfoRecord| record.ts).await
}
fn extract_info_record_from_vod_info(content: &str) -> Option<(u32, InputVodInfoRecord)> {
let doc = serde_json::from_str::<Map<String, Value>>(content).ok()?;
let movie_data = doc.get(crate::model::XC_TAG_VOD_INFO_MOVIE_DATA)?.as_object()?;
let provider_id = get_u32_from_serde_value(
movie_data.get(crate::model::XC_TAG_VOD_INFO_STREAM_ID)?,
)?;
let added = movie_data
.get(crate::model::XC_TAG_VOD_INFO_ADDED)
.and_then(get_u64_from_serde_value)
.unwrap_or(0);
let info_section = doc.get(crate::model::XC_TAG_VOD_INFO_INFO)?.as_object()?;
let tmdb_id = info_section
.get(crate::model::XC_TAG_VOD_INFO_TMDB_ID)
.and_then(get_u32_from_serde_value)
.filter(|&id| id != 0)
.or_else(|| {
info_section
.get(crate::model::XC_TAG_VOD_INFO_TMDB)
.and_then(get_u32_from_serde_value)
})
.unwrap_or(0);
let release_date = info_section
.get(crate::model::XC_TAG_VOD_INFO_RELEASEDATE)
.and_then(get_string_from_serde_value);
Some((provider_id, InputVodInfoRecord {
tmdb_id,
ts: added,
release_date,
}))
}
fn should_update_vod_info(pli: &mut PlaylistItem, processed_provider_ids: &HashMap<u32, u64>) -> (bool, u32, u64) {
should_update_info(pli, processed_provider_ids, crate::model::XC_TAG_VOD_INFO_ADDED)
}
pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, fpl: &mut FetchedPlaylist<'_>) {
pub async fn playlist_resolve_vod(app_config: &Arc<AppConfig>, client: &reqwest::Client, target: &ConfigTarget, errors: &mut Vec<TuliproxError>, fpl: &mut FetchedPlaylist<'_>) {
let (resolve_movies, resolve_delay) = get_resolve_vod_options(target, fpl);
if !resolve_movies { return; }
// TODO read existing WAL File and import it to avoid duplicate requests
// we cant write to the indexed-document directly because of the write lock and time-consuming operation.
// All readers would be waiting for the lock and the app would be unresponsive.
// We collect the content into a wal file and write it once we collected everything.
let config = app_config.config.load();
let Some((mut wal_content_file, mut wal_record_file, wal_content_path, wal_record_path))
= create_resolve_info_wal_files(&config, fpl.input, XtreamCluster::Video).await
else { return; };
let mut processed_info_ids: HashMap<u32, u64> = read_processed_vod_info_ids(app_config, errors, fpl).await;
let mut fetched_in_run: HashSet<u32> = HashSet::new();
let mut content_writer = utils::async_file_writer(&mut wal_content_file);
let mut record_writer = utils::async_file_writer(&mut wal_record_file);
let mut content_updated = false;
let input = fpl.input;
let working_dir = &app_config.config.load().working_dir;
let storage_path = match get_input_storage_path(&input.name, working_dir) {
Ok(storage_path) => storage_path,
Err(err) => {
error!("Can't resolve vod, input storage directory for input '{}' failed: {err}", input.name);
return;
}
};
// LocalVideo entries are not resolved!
// TODO merge both filters to one
let vod_info_count = fpl.playlistgroups.iter()
.flat_map(|plg| &plg.channels)
.filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Video && pli.header.item_type == PlaylistItemType::Video).count();
.filter(|pli| pli.header.xtream_cluster == XtreamCluster::Video
&& pli.header.item_type == PlaylistItemType::Video
&& !pli.has_details()).count();
info!("Found {vod_info_count} vod info to resolve");
let mut last_log_time = Instant::now();
let mut processed_vod_info_count = 0;
let mut write_counter = 0usize;
for plg in &mut fpl.playlistgroups {
for pli in &mut plg.channels {
// LocalVideo files are not resolved
if pli.header.xtream_cluster != XtreamCluster::Video || pli.header.item_type != PlaylistItemType::Video {
if pli.header.xtream_cluster != XtreamCluster::Video
|| pli.header.item_type != PlaylistItemType::Video
|| pli.has_details() {
continue;
}
let (should_update, provider_id, _ts) = should_update_vod_info(pli, &processed_info_ids);
if should_update && provider_id != 0 && fetched_in_run.insert(provider_id) {
if let Some(content) = if pli.get_item_type().is_local() {
pli.header.additional_properties.as_ref().map(ToString::to_string)
} else {
playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await
} {
let normalized_content = normalize_json_content(content);
let normalized_str: &str = &normalized_content;
if let Some((provider_id, info_record)) = extract_info_record_from_vod_info(normalized_str) {
let ts = info_record.ts;
handle_error_and_return!(write_vod_info_to_wal_file(provider_id, normalized_str, &info_record, &mut content_writer, &mut record_writer).await,
|err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}"))));
processed_info_ids.insert(provider_id, ts);
content_updated = true;
write_counter += normalized_str.len();
// periodic flush to bound BufWriter memory
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
if let Err(err) = content_writer.flush().await {
errors.push(notify_err!(format!("Failed periodic flush of wal content writer {err}")));
}
if let Err(err) = record_writer.flush().await {
errors.push(notify_err!(format!("Failed periodic flush of wal record writer {err}")));
let Some(provider_id) = pli.get_provider_id() else { continue; };
processed_vod_info_count += 1;
if provider_id != 0 {
if let Some(content) = playlist_resolve_download_playlist_item(client, pli, fpl.input, errors, resolve_delay, XtreamCluster::Video).await {
match serde_json::from_str::<XtreamVideoInfo>(&content) {
Ok(info) => {
let video_stream_props = VideoStreamProperties::from_info(&info, pli);
if let Err(err) = persist_input_vod_info(app_config, &storage_path, pli.header.xtream_cluster, &input.name, provider_id, &video_stream_props).await {
error!("Failed to persist VOD info for provider_id {provider_id}: {err}");
}
// This makes the data available for subsequent processing steps like STRM export.
pli.header.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props)));
}
// Update in-memory playlist items with the newly fetched vod info.
// This makes the data available for subsequent processing steps like STRM export.
if let Ok(value) = serde_json::from_str::<Value>(normalized_str) {
if let Some(info_content) = value.get("info") {
if let Ok(raw_info) = serde_json::to_string(info_content) {
pli.header.additional_properties = serde_json::from_str::<Box<RawValue>>(&raw_info).ok();
}
}
Err(err) => {
error!("Failed to parse video info for provider_id {provider_id}: {err}");
}
}
}
}
if log_enabled!(Level::Info) {
processed_vod_info_count += 1;
if last_log_time.elapsed().as_secs() >= 30 {
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
last_log_time = Instant::now();
}
if log_enabled!(Level::Info) && last_log_time.elapsed().as_secs() >= 30 {
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
last_log_time = Instant::now();
}
}
}
info!("resolved {processed_vod_info_count}/{vod_info_count} vod info");
if content_updated {
// TODO better approach for transactional updates is multiplexed WAL file.
// final flush & sync with proper error handling
handle_error!(content_writer.flush().await,
|err| errors.push(notify_err!(format!("Failed to resolve vod, could not write to wal file {err}"))));
handle_error!(record_writer.flush().await,
|err| errors.push(notify_err!(format!("Failed to resolve vod tmdb, could not write to wal file {err}"))));
handle_error!(content_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync vod info to wal file {err}"))));
handle_error!(record_writer.get_ref().sync_all().await, |err| errors.push(notify_err!(format!("Failed to sync vod info record to wal file {err}"))));
// drop writers and files to release handles
drop(content_writer);
drop(record_writer);
drop(wal_content_file);
drop(wal_record_file);
handle_error!(xtream_update_input_info_file(app_config, fpl.input, &wal_content_path, XtreamCluster::Video).await,
|err| errors.push(err));
handle_error!(xtream_update_input_vod_record_from_wal_file(app_config, fpl.input, &wal_record_path).await,
|err| errors.push(err));
}
}
backend/src/ptt/constants.rs
+83
View File
@@ -0,0 +1,83 @@
use std::sync::LazyLock;
use regex::Regex;
use fancy_regex::{Regex as FancyRegex};
pub const BRACKETS: &[(&str, &str)] = &[("{", "}"), ("[", "]"), ("(", ")")];
pub const NON_ENGLISH_RANGES: &str = "\u{3040}-\u{30ff}\u{3400}-\u{4dbf}\u{4e00}-\u{9fff}\u{f900}-\u{faff}\u{ff66}-\u{ff9f}\u{0400}-\u{04ff}\u{0600}-\u{06ff}\u{0750}-\u{077f}\u{0c80}-\u{0cff}\u{0d00}-\u{0d7f}\u{0e00}-\u{0e7f}";
pub struct PttConstants {
pub movie_regex: Regex,
pub russian_cast_regex_fancy: FancyRegex,
pub alt_titles_regex: Regex,
pub before_title_match_regex: Regex,
pub not_only_non_english_regex: FancyRegex,
pub not_allowed_symbols_at_start_and_end: FancyRegex,
pub remaining_not_allowed_symbols_at_start_and_end: Regex,
pub redundant_symbols_at_end: Regex,
pub empty_brackets_regex: Regex,
pub parantheses_without_content: Regex,
pub star_regex_1: Regex,
pub star_regex_2: Regex,
pub mp3_regex: Regex,
pub spacing_regex: Regex,
pub special_char_spacing: Regex,
pub sub_pattern: Regex,
pub integer: Regex,
pub word: Regex,
pub dot: Regex,
pub months: Vec<(Regex, &'static str)>,
}
pub static PTT_CONSTANTS: LazyLock<PttConstants> = LazyLock::new(|| {
PttConstants {
movie_regex: Regex::new(r"(?i)[\[(]movie[)\]]").unwrap(),
russian_cast_regex_fancy: FancyRegex::new(
r"(?i)\([^)]*[\u0400-\u04ff][^)]*\)$|(?<=\/.*)\(.*\)$",
)
.unwrap(),
alt_titles_regex: Regex::new(&format!(
r"(?i)[^/|(]*[{NON_ENGLISH_RANGES}][^/|]*[/|]|[/|][^/|(]*[{NON_ENGLISH_RANGES}][^/|]*"
))
.unwrap(),
before_title_match_regex: Regex::new(r"^(?:\[([^\]]+)\])").unwrap(),
not_only_non_english_regex: FancyRegex::new(&format!(
r"(?i)([a-zA-Z][^{NON_ENGLISH_RANGES}]+)([{NON_ENGLISH_RANGES}].*[{NON_ENGLISH_RANGES}])|[{NON_ENGLISH_RANGES}].*[{NON_ENGLISH_RANGES}](?=[^{NON_ENGLISH_RANGES}]+[a-zA-Z])"
)).unwrap(),
not_allowed_symbols_at_start_and_end: FancyRegex::new(&format!(
r"(?i)^[^\w{NON_ENGLISH_RANGES}#\x5B【★]+|[ \-:/\x5C\x5B|{{(#$&^]+$"
))
.unwrap(),
remaining_not_allowed_symbols_at_start_and_end: Regex::new(
&format!(r"^[^\w{NON_ENGLISH_RANGES}#]+|]$")
)
.unwrap(),
redundant_symbols_at_end: Regex::new(r"[ \-:./\\]+$").unwrap(),
empty_brackets_regex: Regex::new(r"\(\s*\)|\[\s*\]|\{\s*\}").unwrap(),
parantheses_without_content: Regex::new(r"\(\W*\)|\[\W*\]|\{\W*\}").unwrap(),
star_regex_1: Regex::new(r"^[\[【★].*[\]】★][ .]?(.+)").unwrap(),
star_regex_2: Regex::new(r"(.+)[ .]?[\[【★].*[\]】★]$").unwrap(),
mp3_regex: Regex::new(r"(?i)\bmp3$").unwrap(),
spacing_regex: Regex::new(r"\s+").unwrap(),
special_char_spacing: Regex::new(r"[\-\+\_\{\}\[\]]\W{2,}").unwrap(),
sub_pattern: Regex::new(r"_+").unwrap(),
integer: Regex::new(r"\d+").unwrap(),
word: Regex::new(r"\W+").unwrap(),
dot: Regex::new(r"\.").unwrap(),
months: vec! [
(Regex::new(r"(?i)\bJanu\b").unwrap(), "Jan"),
(Regex::new(r"(?i)\bFebr\b").unwrap(), "Feb"),
(Regex::new(r"(?i)\bMarc\b").unwrap(), "Mar"),
(Regex::new(r"(?i)\bApri\b").unwrap(), "Apr"),
(Regex::new(r"(?i)\bMay\b").unwrap(), "May"),
(Regex::new(r"(?i)\bJune\b").unwrap(), "Jun"),
(Regex::new(r"(?i)\bJuly\b").unwrap(), "Jul"),
(Regex::new(r"(?i)\bAugu\b").unwrap(), "Aug"),
(Regex::new(r"(?i)\bSept\b").unwrap(), "Sep"),
(Regex::new(r"(?i)\bOcto\b").unwrap(), "Oct"),
(Regex::new(r"(?i)\bNove\b").unwrap(), "Nov"),
(Regex::new(r"(?i)\bDece\b").unwrap(), "Dec"),
]
}
});
backend/src/ptt/handlers.rs
+4708
View File
File diff suppressed because it is too large Load Diff
backend/src/ptt/mod.rs
+20
View File
@@ -0,0 +1,20 @@
use std::sync::LazyLock;
use crate::ptt::parser::PttParser;
mod handlers;
mod models;
mod parser;
mod transformers;
mod constants;
pub use models::PttMetadata;
static PTT_PARSER: LazyLock<PttParser> = LazyLock::new(|| {
let mut parser = PttParser::new();
handlers::add_defaults(&mut parser);
parser
});
pub fn ptt_parse_title(title: &str) -> PttMetadata {
PTT_PARSER.parse(title, false)
}
backend/src/ptt/models.rs
+95
View File
@@ -0,0 +1,95 @@
use serde::{Deserialize, Serialize};
#[allow(clippy::struct_excessive_bools)]
#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)]
pub struct PttMetadata {
pub title: String,
pub seasons: Vec<u32>,
pub episodes: Vec<u32>,
pub languages: Vec<String>,
#[serde(default)]
pub year: Option<u32>,
#[serde(default)]
pub tmdb: Option<u32>,
#[serde(default)]
pub tvdb: Option<u32>,
#[serde(default)]
pub resolution: Option<String>,
#[serde(default)]
pub quality: Option<String>,
#[serde(default)]
pub codec: Option<String>,
pub audio: Vec<String>,
#[serde(default)]
pub group: Option<String>,
#[serde(default)]
pub region: Option<String>,
#[serde(default)]
pub container: Option<String>,
#[serde(default)]
pub size: Option<String>,
pub networks: Vec<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub extended: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub hardcoded: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub proper: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub repack: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub retail: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub remastered: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub unrated: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub uncensored: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub documentary: bool,
#[serde(default)]
pub episode_code: Option<String>,
#[serde(default)]
pub date: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub adult: bool,
#[serde(default)]
pub site: Option<String>,
#[serde(default)]
pub bit_depth: Option<String>,
#[serde(default)]
pub hdr: Vec<String>,
#[serde(default)]
pub channels: Vec<String>,
#[serde(default)]
pub volumes: Vec<i32>,
#[serde(default)]
pub edition: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub trash: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub upscaled: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub convert: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub commentary: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub subbed: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub dubbed: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub is_3d: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub complete: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub ppv: bool,
#[serde(default)]
pub country: Option<String>,
#[serde(default)]
pub bitrate: Option<String>,
#[serde(default)]
pub extension: Option<String>,
}
backend/src/ptt/parser.rs
+249
View File
@@ -0,0 +1,249 @@
use std::cmp::min;
use std::collections::HashMap;
use fancy_regex::{Regex as FancyRegex, Match};
use crate::ptt::constants::{BRACKETS, PTT_CONSTANTS};
use crate::ptt::models::PttMetadata;
#[derive(Debug, Clone)]
pub struct MatchInfo {
pub raw_match: String,
pub match_index: usize,
pub remove: bool,
pub skip_from_title: bool,
}
pub struct ParseContext {
pub title: String,
pub result: PttMetadata,
pub matched: HashMap<String, MatchInfo>,
}
pub type HandlerFn = Box<dyn Fn(&mut ParseContext) -> Option<MatchInfo> + Send + Sync>;
#[derive(Default)]
pub struct PttParser {
handlers: Vec<(String, HandlerFn)>,
}
impl PttParser {
pub fn new() -> Self {
Self::default()
}
pub fn add_handler_fn(&mut self, name: &str, handler: HandlerFn) {
self.handlers.push((name.to_string(), handler));
}
#[allow(clippy::needless_pass_by_value)]
pub fn add_handler<T, F, S>(
&mut self,
name: &str,
regex: FancyRegex,
transformer: F,
setter: S,
options: HandlerOptions,
) where
T: Clone + Send + Sync + 'static,
F: Fn(&str) -> T + Send + Sync + 'static,
S: Fn(&mut PttMetadata, T) + Send + Sync + 'static,
{
let name_owned = name.to_string();
let name_clone = name.to_string();
let handler = Box::new(move |context: &mut ParseContext| -> Option<MatchInfo> {
if options.skip_if_already_found && context.matched.contains_key(&name_owned) {
return None;
}
let match_result: Option<Match> = regex.find(&context.title).ok().flatten();
if let Some(m) = match_result {
let raw_match = m.as_str().to_string();
let match_index = m.start();
let clean_match = if regex.captures_len() > 1 {
regex
.captures(&context.title)
.ok()
.flatten()
.and_then(|c| c.get(1).map(|g| g.as_str().to_string()))
.unwrap_or_else(|| raw_match.clone())
} else {
raw_match.clone()
};
let before_match_opt = PTT_CONSTANTS.before_title_match_regex.find(&context.title);
let before_title_matched = if let Some(bm) = before_match_opt {
let bm_str = bm.as_str();
if bm_str.len() > 2 && bm_str.starts_with('[') && bm_str.ends_with(']') {
let content = &bm_str[1..bm_str.len() - 1];
content.contains(&raw_match)
} else {
false
}
} else {
false
};
let current_skip_from_title = before_title_matched || options.skip_from_title;
let transformed_value = transformer(&clean_match);
if options.skip_if_first {
let other_matches_exist = !context.matched.is_empty();
if other_matches_exist {
let current_start = match_index;
let is_before_all = context
.matched
.values()
.all(|m| current_start < m.match_index);
if is_before_all {
return None;
}
}
}
setter(&mut context.result, transformed_value);
let info = MatchInfo {
raw_match,
match_index,
remove: options.remove,
skip_from_title: current_skip_from_title,
};
context.matched.insert(name_owned.clone(), info.clone());
return Some(info);
}
None
});
self.handlers.push((name_clone, handler));
}
pub fn parse(&self, raw_title: &str, _translate_languages: bool) -> PttMetadata {
let mut context = ParseContext {
title: raw_title.to_string(),
result: PttMetadata::default(),
matched: HashMap::new(),
};
context.title = PTT_CONSTANTS.sub_pattern.replace_all(raw_title, " ").to_string();
let mut end_of_title = context.title.len();
for (_name, handler) in &self.handlers {
if let Some(match_info) = handler(&mut context) {
let match_index = match_info.match_index;
let raw_len = match_info.raw_match.len();
if match_info.remove && match_index + raw_len <= context.title.len() {
context
.title
.replace_range(match_index..match_index + raw_len, "");
}
if !match_info.skip_from_title && match_index > 1 && match_index < end_of_title {
end_of_title = match_index;
}
if match_info.remove
&& match_info.skip_from_title
&& match_index < end_of_title
&& end_of_title >= raw_len
{
end_of_title -= min(raw_len, end_of_title - match_index);
}
}
}
let mut result = context.result;
result.seasons.sort_unstable();
result.seasons.dedup();
result.episodes.sort_unstable();
result.episodes.dedup();
result.languages.sort_unstable();
result.languages.dedup();
let final_title = if end_of_title <= context.title.len() {
context.title[..end_of_title].to_string()
} else {
context.title
};
result.title = clean_title(&final_title);
result
}
}
pub fn clean_title(raw_title: &str) -> String {
let mut title = raw_title.replace('_', " ");
title = PTT_CONSTANTS.movie_regex.replace_all(&title, "").to_string();
title = PTT_CONSTANTS
.not_allowed_symbols_at_start_and_end
.replace_all(&title, "")
.to_string();
title = PTT_CONSTANTS
.russian_cast_regex_fancy
.replace_all(&title, "")
.to_string();
title = PTT_CONSTANTS.star_regex_1.replace_all(&title, "$1").to_string();
title = PTT_CONSTANTS.star_regex_2.replace_all(&title, "$1").to_string();
title = PTT_CONSTANTS.alt_titles_regex.replace_all(&title, "").to_string();
title = PTT_CONSTANTS
.not_only_non_english_regex
.replace_all(&title, "$1")
.to_string();
title = PTT_CONSTANTS
.remaining_not_allowed_symbols_at_start_and_end
.replace_all(&title, "")
.to_string();
title = PTT_CONSTANTS.empty_brackets_regex.replace_all(&title, "").to_string();
title = PTT_CONSTANTS.mp3_regex.replace_all(&title, "").to_string();
title = PTT_CONSTANTS
.parantheses_without_content
.replace_all(&title, "")
.to_string();
title = PTT_CONSTANTS
.special_char_spacing
.replace_all(&title, "")
.to_string();
for (open, close) in BRACKETS {
let open_count = title.matches(open).count();
let close_count = title.matches(close).count();
if open_count != close_count {
title = title.replace(open, "").replace(close, "");
}
}
if !title.trim().contains(' ') && title.contains('.') {
title = PTT_CONSTANTS.dot.replace_all(&title, " ").to_string();
}
title = PTT_CONSTANTS.redundant_symbols_at_end.replace_all(&title, "").to_string();
title = PTT_CONSTANTS.spacing_regex.replace_all(&title, " ").trim().to_string();
title
}
#[allow(clippy::struct_excessive_bools)]
pub struct HandlerOptions {
pub skip_if_already_found: bool,
pub skip_from_title: bool,
pub skip_if_first: bool,
pub remove: bool,
}
impl Default for HandlerOptions {
fn default() -> Self {
Self {
skip_if_already_found: true,
skip_from_title: false,
skip_if_first: false,
remove: false,
}
}
}
backend/src/ptt/transformers.rs
+106
View File
@@ -0,0 +1,106 @@
use chrono::NaiveDate;
use crate::ptt::constants::PTT_CONSTANTS;
pub fn none(input: &str) -> String {
input.to_string()
}
pub fn value(val: &str) -> String {
val.to_string()
}
pub fn boolean(_: &str) -> bool {
true
}
pub fn uinteger(input: &str) -> Option<u32> {
input.parse::<u32>().ok()
}
pub fn first_uinteger(input: &str) -> Option<u32> {
PTT_CONSTANTS.integer.find(input).and_then(|m| m.as_str().parse::<u32>().ok())
}
pub fn lowercase(input: &str) -> String {
input.to_lowercase()
}
pub fn uppercase(input: &str) -> String {
input.to_uppercase()
}
pub fn convert_months(date_str: &str) -> String {
let mut result = date_str.to_string();
for (re, short) in &PTT_CONSTANTS.months {
result = re.replace_all(&result, *short).to_string();
}
result
}
pub fn date(input: &str, formats: &[&str]) -> Option<String> {
let sanitized = PTT_CONSTANTS.word
.replace_all(input, " ")
.trim()
.to_string();
let sanitized = convert_months(&sanitized);
for fmt in formats {
if let Ok(dt) = NaiveDate::parse_from_str(&sanitized, fmt) {
return Some(dt.format("%Y-%m-%d").to_string());
}
}
None
}
macro_rules! range_func {
($name:ident, $t:ty) => {
pub fn $name(input: &str) -> Option<Vec<$t>> {
let numbers: Vec<$t> = PTT_CONSTANTS.integer
.find_iter(input)
.filter_map(|mat| mat.as_str().parse().ok())
.collect();
if numbers.len() == 2 && numbers[0] < numbers[1] {
return Some((numbers[0]..=numbers[1]).collect());
}
match numbers.len() {
len if len > 2 => {
if numbers.windows(2).all(|w| w[0] + 1 == w[1]) {
Some(numbers)
} else {
None
}
}
1 => Some(numbers),
_ => None,
}
}
};
}
range_func!(range_i32, i32);
range_func!(range_u32, u32);
pub fn transform_resolution(input: &str) -> String {
let lower = input.to_lowercase();
if lower.contains("2160") || lower.contains("4k") {
return "2160p".to_string();
}
if lower.contains("1440") || lower.contains("2k") {
return "1440p".to_string();
}
if lower.contains("1080") {
return "1080p".to_string();
}
if lower.contains("720") {
return "720p".to_string();
}
if lower.contains("480") {
return "480p".to_string();
}
if lower.contains("360") {
return "360p".to_string();
}
if lower.contains("240") {
return "240p".to_string();
}
input.to_string()
}
backend/src/repository/bplustree.rs
+2180 -389
View File
File diff suppressed because it is too large Load Diff
backend/src/repository/indexed_document.rs
-627
View File
@@ -1,627 +0,0 @@
use std::fmt::Debug;
use std::fs::{File};
use std::io::{BufReader, Error, ErrorKind, Read, Seek, SeekFrom, Write};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery};
use log::error;
use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;
use shared::error::{str_to_io_error, to_io_error};
use crate::utils;
use crate::utils::{bincode_deserialize, bincode_serialize, IO_BUFFER_SIZE};
const BLOCK_SIZE: usize = 4096;
const LEN_SIZE: usize = 4;
pub(in crate::repository) type OffsetPointer = u32;
type SizeType = u32;
pub type IndexedDocumentIndex<K> = BPlusTree<K, OffsetPointer>;
pub(in crate::repository) struct IndexedDocument {}
impl IndexedDocument {
pub(in crate::repository) fn read_fragmentation<R: Read + Seek>(file: &mut R) -> std::io::Result<bool> {
file.seek(SeekFrom::Start(0))?;
let mut bool_bytes = [0u8];
file.read_exact(&mut bool_bytes)?;
Ok(u8::from_le_bytes(bool_bytes) == 1)
}
pub(in crate::repository) fn write_fragmentation<W: Write + Seek>(file: &mut W, fragmented: bool) -> std::io::Result<()> {
file.seek(SeekFrom::Start(0))?;
let fragmented_byte = if fragmented { 1u8.to_le_bytes() } else { 0u8.to_le_bytes() };
file.write_all(&fragmented_byte)
}
pub(in crate::repository) fn read_content_size<R: Read + Seek>(reader: &mut R) -> Result<usize, Error>
{
let mut size_bytes = [0u8; LEN_SIZE];
reader.read_exact(&mut size_bytes)?;
let buf_size = SizeType::from_le_bytes(size_bytes) as usize;
Ok(buf_size)
}
pub(in crate::repository) fn get_offset<K>(index_path: &Path, doc_id: &K) -> Result<u64, Error>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
match BPlusTreeQuery::<K, OffsetPointer>::try_new(index_path) {
Ok(mut tree) => {
tree.query(doc_id).map_or_else(|| Err(Error::new(ErrorKind::NotFound, format!("doc_id not found {doc_id:?}"))), |offset| Ok(u64::from(offset)))
}
Err(err) => Err(err)
}
}
}
////////////////////////////////////////////////////////
///
/// `IndexedDocumentWriter`
///
////////////////////////////////////////////////////////
/**
* Creates two files,
* - content
* - index
*
* Layout of content file record is:
* - content-size (u32) + content (deflate)
*
* index file is a bplustree
*/
pub(in crate::repository) struct IndexedDocumentWriter<K>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
main_path: PathBuf,
index_path: PathBuf,
main_file: File,
main_offset: OffsetPointer,
index_tree: IndexedDocumentIndex<K>,
dirty: bool,
fragmented: bool,
}
impl<K> IndexedDocumentWriter<K>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
fn new_with_mode(main_path: PathBuf, index_path: PathBuf, append: bool) -> Result<Self, Error> {
let append_mode = append && main_path.exists();
let mut main_file = if append_mode {
utils::open_read_write_file(&main_path)
} else {
utils::create_new_file_for_read_write(&main_path)
}?;
// Retrieve file size and convert to `u32` for `main_offset`, if possible
let mut main_offset = main_file
.metadata()
.and_then(|meta| SizeType::try_from(meta.len()).map_err(to_io_error))
.unwrap_or(0);
let mut fragmented = false;
if main_offset == 0 {
IndexedDocument::write_fragmentation(&mut main_file, false)?;
main_offset = 1;
} else {
fragmented = IndexedDocument::read_fragmentation(&mut main_file)?;
}
// Initialize the index tree (BPlusTree) - either by deserializing an existing one or creating a new one
let index_tree = if append_mode && index_path.exists() {
IndexedDocumentIndex::<K>::load(&index_path).unwrap_or_else(|err| {
error!("Failed to load index {}: {err}", index_path.display());
IndexedDocumentIndex::<K>::new()
})
} else {
IndexedDocumentIndex::<K>::new()
};
Ok(Self {
main_path,
index_path,
main_file,
main_offset,
index_tree,
dirty: false,
fragmented,
})
}
pub fn new(main_path: PathBuf, index_path: PathBuf) -> Result<Self, Error> {
Self::new_with_mode(main_path, index_path, false)
}
pub fn new_append(main_path: PathBuf, index_path: PathBuf) -> Result<Self, Error> {
Self::new_with_mode(main_path, index_path, true)
}
pub fn store(&mut self) -> std::io::Result<()> {
if self.dirty {
self.dirty = false;
match self.index_tree.store(&self.index_path) {
Ok(written_bytes) => {
if written_bytes > 0 {
self.main_file.flush()
} else {
Ok(())
}
}
Err(err) => { Err(err) }
}
} else {
Ok(())
}
}
pub fn write_doc<T>(&mut self, doc_id: K, doc: &T) -> Result<(), Error>
where
T: ?Sized + serde::Serialize,
{
// Determine target position (overwrite in place or append)
if let Some(&offset) = self.index_tree.query(&doc_id) {
// Existing entry: check size and choose strategy
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
let old_size = IndexedDocument::read_content_size(&mut self.main_file)?;
// Prepare new payload (bincode utils, compatible with project)
let encoded = bincode_serialize(doc)
.map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?;
let new_size: SizeType = SizeType::try_from(encoded.len() as u64)
.map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?;
if usize::try_from(new_size).unwrap_or(usize::MAX) > old_size {
// Does not fit -> append at end of file (mark fragmentation)
if !self.fragmented {
self.fragmented = true;
IndexedDocument::write_fragmentation(&mut self.main_file, true)?;
}
self.main_file.seek(SeekFrom::End(0))?;
// We write: length + payload
self.dirty = true;
self.main_file.write_all(&new_size.to_le_bytes())?;
self.main_file.write_all(&encoded)?;
self.index_tree.insert(doc_id, self.main_offset);
// Increase main_offset: LEN_SIZE + new_size
let written_bytes_u64 = (LEN_SIZE as u64) + u64::from(new_size);
let written_bytes: SizeType = SizeType::try_from(written_bytes_u64)
.map_err(|e| str_to_io_error(&format!("Written byte count overflow for {}: {e}", self.main_path.display())))?;
self.main_offset = self.main_offset.checked_add(written_bytes)
.ok_or_else(|| str_to_io_error(&format!("main_offset overflow while appending to {}", self.main_path.display())))?;
return Ok(());
}
// Fits (<=): overwrite at the same position
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
self.dirty = true;
self.main_file.write_all(&new_size.to_le_bytes())?;
self.main_file.write_all(&encoded)?;
return Ok(());
}
// New entry -> append
self.main_file.seek(SeekFrom::End(0))?;
// Determine size and write it, then write payload
let encoded = bincode_serialize(doc)
.map_err(|e| str_to_io_error(&format!("Failed to serialize document for {}: {e}", self.main_path.display())))?;
let new_size: SizeType = SizeType::try_from(encoded.len() as u64)
.map_err(|e| str_to_io_error(&format!("Encoded document size does not fit into u32 for {}: {e}", self.main_path.display())))?;
self.dirty = true;
self.main_file
.write_all(&new_size.to_le_bytes())
.map_err(|e| str_to_io_error(&format!("Failed to write length prefix to {}: {e}", self.main_path.display())))?;
self.main_file
.write_all(&encoded)
.map_err(|e| str_to_io_error(&format!("Failed to write document payload to {}: {e}", self.main_path.display())))?;
self.index_tree.insert(doc_id, self.main_offset);
let written_bytes_u64 = (LEN_SIZE as u64) + u64::from(new_size);
let written_bytes: SizeType = SizeType::try_from(written_bytes_u64)
.map_err(|e| str_to_io_error(&format!("Written byte count overflow for {}: {e}", self.main_path.display())))?;
self.main_offset = self.main_offset.checked_add(written_bytes)
.ok_or_else(|| str_to_io_error(&format!("main_offset overflow while appending to {}", self.main_path.display())))?;
Ok(())
}
}
impl<K> Drop for IndexedDocumentWriter<K>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
fn drop(&mut self) {
let _ = self.store();
}
}
////////////////////////////////////////////////////////
///
/// `IndexedDocumentReader`
///
////////////////////////////////////////////////////////
pub struct IndexedDocumentReader<K, T>
where
T: serde::de::DeserializeOwned,
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
main_file: BufReader<File>,
index_tree: IndexedDocumentIndex<K>,
buffer: Vec<u8>,
t_type: PhantomData<T>,
}
impl<K, T> IndexedDocumentReader<K, T>
where
T: serde::de::DeserializeOwned,
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
pub fn new(main_path: &Path, index_path: &Path) -> Result<Self, Error> {
if main_path.exists() && index_path.exists() {
let main_file = utils::open_readonly_file(main_path)?;
let index_tree = IndexedDocumentIndex::<K>::load(index_path)?;
Ok(Self {
// Larger read buffer for sequential access
main_file: BufReader::with_capacity(BLOCK_SIZE * 64, main_file),
index_tree,
buffer: Vec::with_capacity(BLOCK_SIZE),
t_type: PhantomData,
})
} else {
Err(Error::new(ErrorKind::NotFound, format!("File not found {}", main_path.display())))
}
}
pub fn get(&mut self, doc_id: &K) -> Result<T, Error> {
if let Some(offset) = self.index_tree.query(doc_id) {
self.main_file.seek(SeekFrom::Start(u64::from(*offset)))?;
let buf_size = IndexedDocument::read_content_size(&mut self.main_file)?;
self.buffer.resize(buf_size, 0u8);
self.main_file.read_exact(&mut self.buffer[..buf_size])?;
if let Ok(item) = bincode_deserialize::<T>(&self.buffer[..buf_size]) {
return Ok(item);
}
}
Err(Error::new(ErrorKind::NotFound, format!("Entry not found {doc_id:?}")))
}
}
////////////////////////////////////////////////////////
/// `IndexedDocumentIterator`
///
/// Iterator | Sequential access with `has_next` / next
////////////////////////////////////////////////////////
pub(in crate::repository) struct IndexedDocumentIterator<K, T> {
main_path: PathBuf,
main_file: BufReader<File>,
offsets: Vec<OffsetPointer>,
index: usize,
failed: bool,
t_buffer: Vec<u8>,
t_type: PhantomData<T>,
k_type: PhantomData<K>,
}
impl<K, T> IndexedDocumentIterator<K, T>
where
T: serde::de::DeserializeOwned,
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
pub fn new(main_path: &Path, index_path: &Path) -> Result<Self, Error> {
if main_path.exists() && index_path.exists() {
let mut offsets = Vec::<OffsetPointer>::new();
{
let index_tree = IndexedDocumentIndex::<K>::load(index_path)?;
index_tree.traverse(|_, values| offsets.extend(values));
offsets.sort_unstable();
}
match File::open(main_path) {
Ok(file) => {
Ok(Self {
main_path: main_path.to_path_buf(),
// Larger read buffer for sequential iteration
main_file: BufReader::with_capacity(BLOCK_SIZE * 128, file),
offsets,
index: 0,
failed: false,
t_buffer: Vec::with_capacity(BLOCK_SIZE),
t_type: PhantomData,
k_type: PhantomData,
})
}
Err(e) => Err(e)
}
} else {
Err(Error::new(ErrorKind::NotFound, format!("File not found {}", main_path.display())))
}
}
pub fn get_path(&self) -> &Path {
&self.main_path
}
pub const fn has_error(&self) -> bool {
self.failed
}
pub fn has_next(&self) -> bool {
!self.failed && self.index < self.offsets.len()
}
pub fn read_next(&mut self) -> Result<Option<(T, bool)>, Error> {
if !self.has_next() {
return Ok(None);
}
// read content-size
self.main_file.seek(SeekFrom::Start(u64::from(self.offsets[self.index])))?;
self.index += 1;
let has_next = self.has_next();
let buf_size: usize = IndexedDocument::read_content_size(&mut self.main_file)?;
// resize buffer if necessary
if self.t_buffer.capacity() < buf_size {
self.t_buffer.reserve(buf_size - self.t_buffer.capacity());
}
self.t_buffer.resize(buf_size, 0u8);
// read content
self.main_file.read_exact(&mut self.t_buffer[0..buf_size])?;
// deserialize buffer
match bincode_deserialize::<T>(&self.t_buffer[0..buf_size]) {
Ok(value) => Ok(Some((value, has_next))),
Err(err) => {
self.failed = true;
Err(str_to_io_error(&format!("Failed to deserialize document {err}")))
}
}
}
}
impl<K, T: serde::de::DeserializeOwned> Iterator for IndexedDocumentIterator<K, T>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
{
type Item = (T, bool);
// Implement the next() method
fn next(&mut self) -> Option<Self::Item> {
if self.has_next() {
if let Ok(value) = self.read_next() {
return value;
}
}
None
}
}
////////////////////////////////////////////////////////
///
/// `IndexedDocumentDirectAccess`
///
////////////////////////////////////////////////////////
pub(in crate::repository) struct IndexedDocumentDirectAccess {}
impl IndexedDocumentDirectAccess {
pub(in crate::repository) fn read_indexed_item<K, T>(main_path: &Path, index_path: &Path, doc_id: &K) -> Result<T, Error>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Debug,
T: serde::de::DeserializeOwned,
{
if main_path.exists() && index_path.exists() {
// get the offset from index
let offset = IndexedDocument::get_offset(index_path, doc_id)?;
let mut main_file = File::open(main_path)?;
main_file.seek(SeekFrom::Start(offset))?;
let buf_size = IndexedDocument::read_content_size(&mut main_file)?;
let mut buffer: Vec<u8> = vec![0; buf_size];
main_file.read_exact(&mut buffer)?;
if let Ok(item) = bincode_deserialize::<T>(&buffer) {
return Ok(item);
}
}
Err(str_to_io_error(&format!("Failed to read item for id {:?} - {}", doc_id, main_path.display())))
}
}
////////////////////////////////////////////////////////
///
/// `IndexedDocumentGarbageCollector`
///
////////////////////////////////////////////////////////
pub(in crate::repository) struct IndexedDocumentGarbageCollector<K> {
main_path: PathBuf,
index_path: PathBuf,
main_file: File,
index_tree: IndexedDocumentIndex<K>,
}
impl<K> IndexedDocumentGarbageCollector<K>
where
K: Ord + Serialize + for<'de> Deserialize<'de> + Clone,
{
pub fn new(main_path: PathBuf, index_path: PathBuf) -> Result<Self, Error> {
if main_path.exists() && index_path.exists() {
// Attempt to open the main file in the specified mode (append or not)
let main_file = utils::open_read_write_file(&main_path)?;
// Retrieve file size and convert to `u32` for `main_file`, if possible
let size = main_file
.metadata()
.and_then(|meta| SizeType::try_from(meta.len()).map_err(to_io_error))
.unwrap_or(0);
if size < 1 {
return Err(Error::new(ErrorKind::UnexpectedEof, format!("File empty main:{}", main_path.display())));
}
// Initialize the index tree (BPlusTree) - by deserializing an existing one
let index_tree = IndexedDocumentIndex::<K>::load(&index_path)?;
Ok(Self {
main_path,
index_path,
main_file,
index_tree,
})
} else {
Err(Error::new(ErrorKind::NotFound, format!("Files not found main:{} index:{}", main_path.display(), index_path.display())))
}
}
pub fn garbage_collect(&mut self) -> Result<(), Error> {
let fragmented = IndexedDocument::read_fragmentation(&mut self.main_file)?;
if !fragmented {
return Ok(());
}
let gc_file = NamedTempFile::new()?;
let gc_path = gc_file.path();
{
let mut key_offset = Vec::<(K, OffsetPointer)>::new();
self.index_tree.traverse(|keys, values| {
keys.iter().zip(values.iter()).for_each(|(key, &offset)| key_offset.push((key.clone(), offset)));
});
// For better disk locality: sort by offset (sequential reads)
key_offset.sort_by_key(|(_, off)| *off);
let mut gc_writer = utils::file_writer(&gc_file);
let fragmented_byte = 0u8.to_le_bytes();
gc_writer.write_all(&fragmented_byte)?;
let mut gc_offset = 1usize; // offset is 1 because of fragment bit
let mut buffer: Vec<u8> = Vec::with_capacity(BLOCK_SIZE);
let mut size_bytes = [0u8; LEN_SIZE];
let mut write_counter = 0usize;
for (key, offset) in key_offset {
// read old content
self.main_file.seek(SeekFrom::Start(u64::from(offset)))?;
self.main_file.read_exact(&mut size_bytes)?;
let buf_size = SizeType::from_le_bytes(size_bytes) as usize;
// ensure buffer capacity
if buffer.capacity() < buf_size {
buffer.reserve(buf_size - buffer.capacity());
}
buffer.resize(buf_size, 0u8);
self.main_file.read_exact(&mut buffer[0..buf_size])?;
gc_writer.write_all(&size_bytes)?;
gc_writer.write_all(&buffer[0..buf_size])?;
write_counter += buf_size;
if write_counter >= IO_BUFFER_SIZE {
write_counter = 0;
gc_writer.flush()?;
}
let pointer = OffsetPointer::try_from(gc_offset).map_err(to_io_error)?;
self.index_tree.insert(key, pointer);
gc_offset += size_bytes.len() + buf_size; // gc_file.stream_position();
}
gc_writer.flush()?;
}
utils::rename_or_copy(gc_path, &self.main_path, false)?;
self.index_tree.store(&self.index_path)?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::io;
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use shared::model::XtreamPlaylistItem;
// use crate::model::XtreamPlaylistItem;
use crate::repository::indexed_document::{IndexedDocumentGarbageCollector, IndexedDocumentIterator, IndexedDocumentWriter};
use crate::utils::resolve_env_var;
// Example usage with a simple struct
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
struct Record {
id: u32,
data: String,
}
#[test]
fn insert_test() -> io::Result<()> {
let main_path = PathBuf::from("/tmp/main.iw");
let index_path = PathBuf::from("/tmp/main.iw.idx");
{
let mut idw = IndexedDocumentWriter::new(main_path.clone(), index_path.clone())?;
for i in 0u32..=500 {
idw.write_doc(i, &Record {
id: i,
data: format!("Entry {i}"),
})?;
}
let size_main_file_1 = std::fs::metadata(&main_path)?.len();
// update same block, file size should not increase
for i in 0u32..=500 {
idw.write_doc(i, &Record {
id: i,
data: format!("E {i}"),
})?;
}
let size_main_file_2 = std::fs::metadata(&main_path)?.len();
assert_eq!(size_main_file_1, size_main_file_2, "Failed, the filesize should be the same");
// fragmentation
for i in 0u32..=500 {
idw.write_doc(i, &Record {
id: i,
data: format!("Entry {}", i + 9000),
})?;
}
let size_main_file_3 = std::fs::metadata(&main_path)?.len();
assert!(size_main_file_1 < size_main_file_3, "Failed, the filesize should be greater");
idw.store()?;
}
{
let size_main_file_4 = std::fs::metadata(&main_path)?.len();
let mut gc = IndexedDocumentGarbageCollector::<u32>::new(main_path.clone(), index_path.clone())?;
gc.garbage_collect()?;
let size_main_file_5 = std::fs::metadata(&main_path)?.len();
assert!(size_main_file_5 < size_main_file_4, "Failed, the filesize should be less");
}
{
let reader = IndexedDocumentIterator::<u32, Record>::new(&main_path, &index_path)?;
let mut i = 0;
for (doc, _) in reader {
assert_eq!(doc.id, i, "Wrong id");
assert_eq!(doc.data, format!("Entry {}", i + 9000), "Wrong data");
i += 1;
}
assert_eq!(501, i, "Wrong number of elements");
}
Ok(())
}
#[test]
fn test_read_xt() -> io::Result<()> {
let main_path = PathBuf::from(resolve_env_var("${env:HOME}/projects/m3u-test/settings/alexyand/data/all_channels/xtream/live.db"));
let index_path = PathBuf::from(resolve_env_var("${env:HOME}/projects/m3u-test/settings/alexyand/data/all_channels/xtream/live.idx"));
let reader = IndexedDocumentIterator::<u32, XtreamPlaylistItem>::new(&main_path, &index_path)?;
for (doc, _has_next) in reader {
if doc.epg_channel_id.is_none() {
println!("{doc:?}");
}
}
Ok(())
}
}
backend/src/repository/m3u_playlist_iterator.rs
+13 -13
View File
@@ -3,8 +3,8 @@ use shared::error::{TuliproxError};
use crate::model::{AppConfig, ProxyUserCredentials};
use crate::model::{ConfigTarget};
use shared::model::{ConfigTargetOptions, M3uPlaylistItem, PlaylistItemType, ProxyType, TargetType, XtreamCluster};
use crate::repository::indexed_document::IndexedDocumentIterator;
use crate::repository::m3u_repository::m3u_get_file_paths;
use crate::repository::bplustree::{BPlusTreeDiskIteratorOwned, BPlusTreeQuery};
use crate::repository::m3u_repository::m3u_get_file_path;
use crate::repository::storage::ensure_target_storage_path;
use crate::repository::storage_const;
use crate::repository::user_repository::user_get_bouquet_filter;
@@ -14,7 +14,7 @@ use std::collections::HashSet;
#[allow(clippy::struct_excessive_bools)]
pub struct M3uPlaylistIterator {
reader: IndexedDocumentIterator<u32, M3uPlaylistItem>,
reader: BPlusTreeDiskIteratorOwned<u32, M3uPlaylistItem>,
base_url: String,
username: String,
password: String,
@@ -40,13 +40,13 @@ impl M3uPlaylistIterator {
let m3u_output = target.get_m3u_output().ok_or_else(|| info_err!(format!("Unexpected failure, missing m3u target output for target {}", target.name)))?;
let config = cfg.config.load();
let target_path = ensure_target_storage_path(&config, target.name.as_str())?;
let (m3u_path, idx_path) = m3u_get_file_paths(&target_path);
let m3u_path = m3u_get_file_path(&target_path);
let file_lock = cfg.file_locks.read_lock(&m3u_path).await;
let reader =
IndexedDocumentIterator::<u32, M3uPlaylistItem>::new(&m3u_path, &idx_path)
.map_err(|err| info_err!(format!("Could not deserialize file {m3u_path:?} - {err}")))?;
let query = BPlusTreeQuery::<u32, M3uPlaylistItem>::try_new(&m3u_path)
.map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {m3u_path:?} - {err}")))?;
let reader = query.disk_iter();
let filter = user_get_bouquet_filter(&config, &user.username, None, TargetType::M3u, XtreamCluster::Live).await;
@@ -116,14 +116,14 @@ impl M3uPlaylistIterator {
let entry = if let Some(set) = &self.filter {
if let Some((current_item, _)) = self.lookup_item.take() {
// Avoid cloning strings while filtering
let next_valid = self.reader.find(|(pli, _)| set.contains(pli.group.as_str()));
self.lookup_item = next_valid;
let next_valid = self.reader.find(|(_, pli)| set.contains(pli.group.as_str()));
self.lookup_item = next_valid.map(|(_k, v)| (v, true)); // has_next handled by iterator usually, but here we just need the item
let has_next = self.lookup_item.is_some();
Some((current_item, has_next))
} else {
let current_item = self.reader.find(|(item, _)| set.contains(item.group.as_str()));
if let Some((item, _)) = current_item {
self.lookup_item = self.reader.find(|(item, _)| set.contains(item.group.as_str()));
let current_item = self.reader.find(|(_, item)| set.contains(item.group.as_str()));
if let Some((_, item)) = current_item {
self.lookup_item = self.reader.find(|(_, item)| set.contains(item.group.as_str())).map(|(_, item)| (item, true));
let has_next = self.lookup_item.is_some();
Some((item, has_next))
} else {
@@ -131,7 +131,7 @@ impl M3uPlaylistIterator {
}
}
} else {
self.reader.next()
self.reader.next().map(|(_, v)| (v, !self.reader.is_empty()))
};
// TODO hls and unknown reverse proxy
backend/src/repository/m3u_repository.rs
+24 -28
View File
@@ -1,7 +1,7 @@
use crate::api::model::AppState;
use crate::model::{AppConfig, ProxyUserCredentials};
use crate::model::{Config, ConfigTarget, M3uTargetOutput};
use crate::repository::indexed_document::{IndexedDocumentDirectAccess, IndexedDocumentIterator, IndexedDocumentWriter};
use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery};
use crate::repository::m3u_playlist_iterator::M3uPlaylistM3uTextIterator;
use crate::repository::storage::get_target_storage_path;
use crate::repository::storage_const;
@@ -9,7 +9,7 @@ use crate::utils;
use shared::error::{create_tuliprox_error, info_err};
use shared::error::{str_to_io_error, TuliproxError, TuliproxErrorKind};
use shared::model::PlaylistItemType;
use shared::model::{M3uPlaylistItem, PlaylistGroup, PlaylistItem};
use shared::model::{M3uPlaylistItem, PlaylistGroup};
use std::io::Error;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -25,10 +25,8 @@ macro_rules! cant_write_result {
}
}
pub fn m3u_get_file_paths(target_path: &Path) -> (PathBuf, PathBuf) {
let m3u_path = target_path.join(PathBuf::from(format!("{}.{}", storage_const::FILE_M3U, storage_const::FILE_SUFFIX_DB)));
let index_path = target_path.join(PathBuf::from(format!("{}.{}", storage_const::FILE_M3U, storage_const::FILE_SUFFIX_INDEX)));
(m3u_path, index_path)
pub fn m3u_get_file_path(target_path: &Path) -> PathBuf {
target_path.join(PathBuf::from(format!("{}.{}", storage_const::FILE_M3U, storage_const::FILE_SUFFIX_DB)))
}
pub fn m3u_get_epg_file_path(target_path: &Path) -> PathBuf {
@@ -88,13 +86,13 @@ pub async fn m3u_write_playlist(
return Ok(());
}
let (m3u_path, idx_path) = m3u_get_file_paths(target_path);
let m3u_path = m3u_get_file_path(target_path);
let m3u_playlist = Arc::new(
new_playlist
.iter()
.flat_map(|pg| &pg.channels)
.filter(|&pli| !matches!(pli.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo))
.map(PlaylistItem::to_m3u)
.map(M3uPlaylistItem::from)
.collect::<Vec<M3uPlaylistItem>>(),
);
@@ -106,23 +104,15 @@ pub async fn m3u_write_playlist(
let playlist = Arc::clone(&m3u_playlist);
let m3u_path_clone = m3u_path.clone();
let idx_path_clone = idx_path.clone();
task::spawn_blocking(move || -> Result<(), TuliproxError> {
let _guard = file_lock;
match IndexedDocumentWriter::new(m3u_path_clone.clone(), idx_path_clone) {
Ok(mut writer) => {
for m3u in playlist.iter() {
writer
.write_doc(m3u.virtual_id, m3u)
.map_err(|err| cant_write_result!(&m3u_path_clone, err))?;
}
writer
.store()
.map_err(|err| cant_write_result!(&m3u_path_clone, err))
}
Err(err) => Err(cant_write_result!(&m3u_path_clone, err)),
let mut tree = BPlusTree::new();
for m3u in playlist.iter() {
tree.insert(m3u.virtual_id, m3u.clone());
}
tree.store(&m3u_path_clone).map_err(|err| cant_write_result!(&m3u_path_clone, err))?;
Ok(())
})
.await
.map_err(|err| create_tuliprox_error!(TuliproxErrorKind::Notify, "failed to write m3u playlist: {} - {err}", m3u_path.display()))??;
@@ -154,22 +144,28 @@ pub async fn m3u_get_item_for_stream_id(stream_id: u32, app_state: &AppState, ta
let cfg: &AppConfig = &app_state.app_config;
let target_path = get_target_storage_path(&cfg.config.load(), target.name.as_str()).ok_or_else(|| str_to_io_error(&format!("Could not find path for target {}", &target.name)))?;
let (m3u_path, idx_path) = m3u_get_file_paths(&target_path);
let m3u_path = m3u_get_file_path(&target_path);
let _file_lock = cfg.file_locks.read_lock(&m3u_path).await;
IndexedDocumentDirectAccess::read_indexed_item::<u32, M3uPlaylistItem>(&m3u_path, &idx_path, &stream_id)
let mut query = BPlusTreeQuery::<u32, M3uPlaylistItem>::try_new(&m3u_path)?;
query.query(&stream_id).ok_or_else(|| str_to_io_error(&format!("Item not found: {stream_id}")))
}
}
pub async fn iter_raw_m3u_playlist(config: &AppConfig, target: &ConfigTarget) -> Option<(utils::FileReadGuard, impl Iterator<Item=(M3uPlaylistItem, bool)>)> {
let target_path = get_target_storage_path(&config.config.load(), target.name.as_str())?;
let (m3u_path, idx_path) = m3u_get_file_paths(&target_path);
if !m3u_path.exists() || !idx_path.exists() {
let m3u_path = m3u_get_file_path(&target_path);
if let Ok(false) = tokio::fs::try_exists(&m3u_path).await {
return None;
}
let file_lock = config.file_locks.read_lock(&m3u_path).await;
match IndexedDocumentIterator::<u32, M3uPlaylistItem>::new(&m3u_path, &idx_path)
.map_err(|err| info_err!(format!("Could not deserialize file {m3u_path:?} - {err}"))) {
Ok(reader) => Some((file_lock, reader)),
match BPlusTreeQuery::<u32, M3uPlaylistItem>::try_new(&m3u_path)
.map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {m3u_path:?} - {err}"))) {
Ok(mut query) => {
let items: Vec<M3uPlaylistItem> = query.iter().map(|(_, v)| v).collect();
let len = items.len();
Some((file_lock, items.into_iter().enumerate().map(move |(i, v)| (v, i < len - 1))))
}
Err(_) => None
}
}
backend/src/repository/mod.rs
+1 -2
View File
@@ -1,8 +1,6 @@
pub mod storage;
pub mod target_id_mapping;
pub mod bplustree;
mod indexed_document;
pub use indexed_document::IndexedDocumentReader;
pub mod playlist_repository;
pub mod m3u_repository;
pub mod xtream_repository;
@@ -12,4 +10,5 @@ pub mod m3u_playlist_iterator;
pub mod xtream_playlist_iterator;
pub mod user_repository;
pub mod storage_const;
mod playlist_scratch;
backend/src/repository/playlist_repository.rs
+116 -48
View File
@@ -1,22 +1,20 @@
use crate::api::model::{AppState, PlaylistM3uStorage, PlaylistStorage, PlaylistStorageState, PlaylistXtreamStorage};
use crate::model::{AppConfig, ConfigTarget, TargetOutput};
use crate::model::{Epg, XtreamSeriesInfo};
use crate::model::{AppConfig, ConfigInput, ConfigTarget, TargetOutput};
use crate::model::Epg;
use crate::processing::processor::playlist::apply_filter_to_playlist;
use crate::repository::bplustree::BPlusTree;
use crate::repository::bplustree::{BPlusTree, BPlusTreeQuery};
use crate::repository::epg_repository::epg_write;
use crate::repository::indexed_document::IndexedDocumentIterator;
use crate::repository::m3u_repository::{m3u_get_file_paths, m3u_write_playlist};
use crate::repository::storage::{ensure_target_storage_path, get_target_id_mapping_file, get_target_storage_path};
use crate::repository::m3u_repository::{m3u_get_file_path, m3u_write_playlist};
use crate::repository::storage::{ensure_target_storage_path, get_input_storage_path, get_target_id_mapping_file, get_target_storage_path};
use crate::repository::strm_repository::write_strm_playlist;
use crate::repository::target_id_mapping::{TargetIdMapping, VirtualIdRecord};
use crate::repository::xtream_repository::{xtream_get_file_paths, xtream_get_storage_path, xtream_write_playlist};
use crate::repository::xtream_repository::{persist_input_xtream_playlist, xtream_get_file_path, xtream_get_storage_path, xtream_write_playlist};
use crate::utils;
use log::info;
use serde_json::value::RawValue;
use shared::create_tuliprox_error;
use shared::error::TuliproxError;
use shared::error::{info_err, TuliproxErrorKind};
use shared::model::{M3uPlaylistItem, PlaylistGroup, PlaylistItemHeader, PlaylistItemType, XtreamCluster, XtreamPlaylistItem};
use shared::model::{InputType, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem};
use shared::utils::{is_dash_url, is_hls_url};
use std::collections::HashMap;
use std::path::Path;
@@ -27,7 +25,12 @@ struct LocalEpisodeKey {
virtual_id: u32,
}
pub async fn persist_playlist(app_config: &AppConfig, playlist: &mut [PlaylistGroup], epg: Option<&Epg>,
pub struct ProviderEpisodeKey {
pub(crate) provider_id: u32,
pub(crate) virtual_id: u32,
}
pub async fn persist_playlist(app_config: &Arc<AppConfig>, playlist: &mut [PlaylistGroup], epg: Option<&Epg>,
target: &ConfigTarget, playlist_state: Option<&Arc<PlaylistStorageState>>) -> Result<(), Vec<TuliproxError>> {
let mut errors = vec![];
let config = &app_config.config.load();
@@ -39,6 +42,7 @@ pub async fn persist_playlist(app_config: &AppConfig, playlist: &mut [PlaylistGr
let (mut target_id_mapping, file_lock) = get_target_id_mapping(app_config, &target_path).await;
let mut local_library_series = HashMap::<String, Vec<LocalEpisodeKey>>::new();
let mut provider_series = HashMap::<String, Vec<ProviderEpisodeKey>>::new();
// Virtual IDs assignment
for group in playlist.iter_mut() {
@@ -63,11 +67,16 @@ pub async fn persist_playlist(app_config: &AppConfig, playlist: &mut [PlaylistGr
let item_type = header.item_type;
header.virtual_id = target_id_mapping.get_and_update_virtual_id(uuid, provider_id, item_type, 0);
assign_local_series_info_episode_key(&mut local_library_series, header, item_type);
if item_type == PlaylistItemType::LocalSeries {
assign_local_series_info_episode_key(&mut local_library_series, header, item_type);
}
if item_type == PlaylistItemType::Series {
assign_provider_series_info_episode_key(&mut provider_series, header, item_type);
}
}
}
rewrite_local_series_info_episode_virtual_id(playlist, &mut local_library_series);
rewrite_series_info_episode_virtual_id(playlist, &local_library_series, &provider_series);
for output in &target.output {
let mut filtered = match output {
@@ -143,29 +152,32 @@ fn assign_local_series_info_episode_key(local_library_series: &mut HashMap<Strin
}
}
fn rewrite_local_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup], local_library_series: &mut HashMap<String, Vec<LocalEpisodeKey>>) {
// assign local series virtual ids
for group in playlist.iter_mut() {
for channel in &mut group.channels {
let header = &mut channel.header;
if header.item_type == PlaylistItemType::LocalSeriesInfo {
if let Some(episode_keys) = local_library_series.get(&header.id) {
if let Some(props) = &header.additional_properties {
let props_str = props.get();
if let Ok(mut series_info) = serde_json::from_str::<XtreamSeriesInfo>(props_str) {
if let Some(episodes) = series_info.episodes.as_mut() {
for episode in episodes.iter_mut() {
for episode_key in episode_keys {
if episode.direct_source == episode_key.path {
episode.id = episode_key.virtual_id;
break;
}
}
}
if let Ok(s) = serde_json::to_string(&series_info) {
header.additional_properties = RawValue::from_string(s).ok();
}
}
fn assign_provider_series_info_episode_key(provider_series: &mut HashMap<String, Vec<ProviderEpisodeKey>>, header: &mut PlaylistItemHeader, item_type: PlaylistItemType) {
// we need to rewrite local series info with the new virtual ids
if item_type == PlaylistItemType::Series {
provider_series
.entry(header.parent_code.clone())
.or_default()
.push(ProviderEpisodeKey {
provider_id: header.get_provider_id().unwrap_or_default(),
virtual_id: header.virtual_id,
});
}
}
#[allow(clippy::implicit_hasher)]
fn rewrite_local_series_info_episode_virtual_id(pli: &mut PlaylistItem, local_library_series: &HashMap<String, Vec<LocalEpisodeKey>>) {
let header = &mut pli.header;
if let Some(episode_keys) = local_library_series.get(&header.id) {
if let Some(StreamProperties::Series(series)) = header.additional_properties.as_mut() {
if let Some(episodes) =
series.details.as_mut().and_then(|d| d.episodes.as_mut())
{
for episode in episodes.iter_mut() {
for episode_key in episode_keys {
if episode.direct_source == episode_key.path {
episode.id = episode_key.virtual_id;
break;
}
}
}
@@ -174,6 +186,46 @@ fn rewrite_local_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup],
}
}
#[allow(clippy::implicit_hasher)]
pub fn rewrite_provider_series_info_episode_virtual_id<P>(pli: &mut P, provider_series: &HashMap<String, Vec<ProviderEpisodeKey>>)
where P: PlaylistEntry
{
if let Some(episode_keys) = provider_series.get(&pli.get_uuid().to_string()) {
if let Some(StreamProperties::Series(series)) = pli.get_additional_properties_mut() {
if let Some(episodes) =
series.details.as_mut().and_then(|d| d.episodes.as_mut())
{
for episode in episodes.iter_mut() {
for episode_key in episode_keys {
if episode.id == episode_key.provider_id {
episode.id = episode_key.virtual_id;
break;
}
}
}
}
}
}
}
fn rewrite_series_info_episode_virtual_id(playlist: &mut [PlaylistGroup],
local_library_series: &HashMap<String, Vec<LocalEpisodeKey>>,
provider_series: &HashMap<String, Vec<ProviderEpisodeKey>>) {
if local_library_series.is_empty() && provider_series.is_empty() {
return;
}
for group in playlist.iter_mut() {
for channel in &mut group.channels {
let item_type = channel.header.item_type;
if item_type == PlaylistItemType::SeriesInfo {
rewrite_provider_series_info_episode_virtual_id(channel, provider_series);
} else if item_type == PlaylistItemType::LocalSeriesInfo {
rewrite_local_series_info_episode_virtual_id(channel, local_library_series);
}
}
}
}
pub async fn get_target_id_mapping(cfg: &AppConfig, target_path: &Path) -> (TargetIdMapping, utils::FileWriteGuard) {
let target_id_mapping_file = get_target_id_mapping_file(target_path);
let file_lock = cfg.file_locks.write_lock(&target_id_mapping_file).await;
@@ -184,20 +236,16 @@ pub async fn get_target_id_mapping(cfg: &AppConfig, target_path: &Path) -> (Targ
async fn load_target_id_mapping_as_tree(app_config: &AppConfig, target_path: &Path, target: &ConfigTarget) -> Result<BPlusTree<u32, VirtualIdRecord>, TuliproxError> {
let target_id_mapping_file = get_target_id_mapping_file(target_path);
let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await;
BPlusTree::<u32, VirtualIdRecord>::load(&target_id_mapping_file).map_err(|err|
create_tuliprox_error!(
TuliproxErrorKind::Info,
"Could not find path for target {} err:{err}", &target.name
))
create_tuliprox_error!(TuliproxErrorKind::Info, "Could not find path for target {} err:{err}", &target.name))
}
async fn load_xtream_playlist_as_tree(app_config: &AppConfig, storage_path: &Path, cluster: XtreamCluster) -> BPlusTree<u32, XtreamPlaylistItem> {
let (main_path, index_path) = xtream_get_file_paths(storage_path, cluster);
let _file_lock = app_config.file_locks.read_lock(&main_path).await;
let xtream_path = xtream_get_file_path(storage_path, cluster);
let _file_lock = app_config.file_locks.read_lock(&xtream_path).await;
let mut tree = BPlusTree::<u32, XtreamPlaylistItem>::new();
if let Ok(reader) = IndexedDocumentIterator::<u32, XtreamPlaylistItem>::new(&main_path, &index_path) {
for (doc, _has_next) in reader {
if let Ok(mut query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path) {
for (_, doc) in query.iter() {
tree.insert(doc.virtual_id, doc);
}
}
@@ -238,11 +286,11 @@ async fn load_m3u_target_storage(app_config: &AppConfig, target: &ConfigTarget)
"Could not find path for target {}", &target.name
))?;
let (main_path, index_path) = m3u_get_file_paths(&target_path);
let _file_lock = app_config.file_locks.read_lock(&main_path).await;
let m3u_path = m3u_get_file_path(&target_path);
let _file_lock = app_config.file_locks.read_lock(&m3u_path).await;
let mut tree = BPlusTree::<u32, M3uPlaylistItem>::new();
if let Ok(reader) = IndexedDocumentIterator::<u32, M3uPlaylistItem>::new(&main_path, &index_path) {
for (doc, _has_next) in reader {
if let Ok(mut query) = BPlusTreeQuery::<u32, M3uPlaylistItem>::try_new(&m3u_path) {
for (_, doc) in query.iter() {
tree.insert(doc.virtual_id, doc);
}
}
@@ -279,3 +327,23 @@ pub async fn load_target_into_memory_cache(app_state: &AppState, target: &Arc<Co
};
}
}
pub async fn persist_input_playlist(app_config: &Arc<AppConfig>, input: &Arc<ConfigInput>, mut playlist: Vec<PlaylistGroup>) -> (Vec<PlaylistGroup>, Option<TuliproxError>) {
playlist.iter_mut().for_each(PlaylistGroup::on_load);
let (result, err) = match input.input_type {
InputType::Xtream | InputType::XtreamBatch => {
let working_dir = &app_config.config.load().working_dir;
let storage_path = match get_input_storage_path(&input.name, working_dir) {
Ok(storage_path) => storage_path,
Err(err) => {
return (playlist, Some(create_tuliprox_error!( TuliproxErrorKind::Info, "Error creating input storage directory for input '{}' failed: {err}", input.name)));
}
};
persist_input_xtream_playlist(app_config, &storage_path, playlist).await
}
_ => (playlist, None),
};
(result, err)
}
backend/src/repository/playlist_scratch.rs
+98
View File
@@ -0,0 +1,98 @@
use indexmap::IndexMap;
use shared::model::XtreamCluster;
pub(in crate::repository) trait WithCapacity {
fn with_capacity(capacity: usize) -> Self;
}
impl<T> WithCapacity for Vec<T> {
fn with_capacity(capacity: usize) -> Self {
Vec::with_capacity(capacity)
}
}
impl<K, V> WithCapacity for IndexMap<K, V> {
fn with_capacity(capacity: usize) -> Self {
IndexMap::with_capacity(capacity)
}
}
pub(in crate::repository) trait IsEmpty {
fn is_empty(&self) -> bool;
}
impl<T> IsEmpty for Vec<T> {
fn is_empty(&self) -> bool {
Vec::is_empty(self)
}
}
impl<K, V> IsEmpty for IndexMap<K, V> {
fn is_empty(&self) -> bool {
IndexMap::is_empty(self)
}
}
pub struct PlaylistScratch<C> {
live: C,
vods: C,
series: C,
}
impl<C: WithCapacity> PlaylistScratch<C> {
pub fn new(capacity: usize) -> Self {
Self {
live: C::with_capacity(capacity),
vods: C::with_capacity(capacity),
series: C::with_capacity(capacity),
}
}
}
impl<C> PlaylistScratch<C> {
pub fn get_mut(&mut self, cluster: XtreamCluster) -> &mut C {
match cluster {
XtreamCluster::Live => &mut self.live,
XtreamCluster::Video => &mut self.vods,
XtreamCluster::Series => &mut self.series,
}
}
// pub fn get(&self, cluster: XtreamCluster) -> &C {
// match cluster {
// XtreamCluster::Live => &self.live,
// XtreamCluster::Video => &self.vods,
// XtreamCluster::Series => &self.series,
// }
// }
pub fn set(&mut self, cluster: XtreamCluster, content: C) {
match cluster {
XtreamCluster::Live => self.live = content,
XtreamCluster::Video => self.vods = content,
XtreamCluster::Series => self.series = content,
}
}
}
impl<C: Default> PlaylistScratch<C> {
pub fn take(&mut self, cluster: XtreamCluster) -> C {
match cluster {
XtreamCluster::Live => std::mem::take(&mut self.live),
XtreamCluster::Video => std::mem::take(&mut self.vods),
XtreamCluster::Series => std::mem::take(&mut self.series),
}
}
}
impl<C: IsEmpty> PlaylistScratch<C> {
pub fn is_empty(&self, cluster: XtreamCluster) -> bool {
match cluster {
XtreamCluster::Live => self.live.is_empty(),
XtreamCluster::Video => self.vods.is_empty(),
XtreamCluster::Series => self.series.is_empty(),
}
}
}
backend/src/repository/storage_const.rs
-12
View File
@@ -1,11 +1,8 @@
pub const FILE_EPG: &str = "epg.xml";
pub(in crate::repository) const FILE_SUFFIX_DB: &str = "db";
pub(in crate::repository) const FILE_SUFFIX_INDEX: &str = "idx";
pub(in crate::repository) const FILE_ID_MAPPING: &str = "id_mapping.db";
pub(in crate::repository) const FILE_STRM: &str = "strm";
pub(in crate::repository) const FILE_M3U: &str = "m3u";
pub const FILE_SUFFIX_WAL: &str = "wal";
pub const M3U_STREAM_PATH: &str = "m3u-stream";
pub const M3U_RESOURCE_PATH: &str = "resource/m3u";
pub const EPG_RESOURCE_PATH: &str = "resource/epg";
@@ -18,14 +15,5 @@ pub(in crate::repository) const USER_LIVE_BOUQUET: &str = "live_bouquet.json";
pub(in crate::repository) const USER_VOD_BOUQUET: &str = "vod_bouquet.json";
pub(in crate::repository) const USER_SERIES_BOUQUET: &str = "series_bouquet.json";
pub(in crate::repository) const API_USER_DB_FILE: &str = "api_user.db";
pub(in crate::repository) const FILE_SERIES_INFO: &str = "series_info";
pub(in crate::repository) const FILE_VOD_INFO: &str = "vod_info";
pub(in crate::repository) const FILE_VOD_INFO_RECORD: &str = "vod_info_record";
pub(in crate::repository) const FILE_SERIES_INFO_RECORD: &str = "series_info_record";
pub(in crate::repository) const FILE_SERIES_EPISODE_RECORD: &str = "series_episode_record";
// pub(in crate::repository) const FILE_SERIES: &str = "series";
pub(in crate::repository) const PATH_XTREAM: &str = "xtream";
pub(in crate::repository) const INFO_REWRITE_FIELDS: &[&str] = &["cover_big", "cover", "cover_tmdb", "movie_image", "tmdb_url", "overview", "kinopoisk_url"];
backend/src/repository/strm_repository.rs
+77 -226
View File
@@ -1,14 +1,10 @@
// Import the new MediaQuality struct
use crate::model::MediaQuality;
use crate::model::XtreamSeriesEpisode;
use crate::model::{ApiProxyServerInfo, AppConfig, ProxyUserCredentials};
use crate::model::{ConfigTarget, StrmTargetOutput};
use crate::repository::bplustree::BPlusTree;
use crate::repository::storage::{ensure_target_storage_path, get_input_storage_path};
use crate::repository::storage::{ensure_target_storage_path};
use crate::repository::storage_const;
use crate::repository::xtream_repository::{xtream_get_record_file_path, InputVodInfoRecord};
use crate::utils;
use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, FileReadGuard, IO_BUFFER_SIZE};
use crate::utils::{async_file_reader, async_file_writer, normalize_string_path, truncate_filename, IO_BUFFER_SIZE};
use chrono::Datelike;
use filetime::{set_file_times, FileTime};
use log::{error, trace};
@@ -16,7 +12,7 @@ use regex::Regex;
use serde::Serialize;
use shared::error::{create_tuliprox_error_result, info_err};
use shared::error::{TuliproxError, TuliproxErrorKind};
use shared::model::{ClusterFlags, FieldGetAccessor, PlaylistGroup, PlaylistItem, PlaylistItemType, StrmExportStyle, UUIDType};
use shared::model::{ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, StrmExportStyle, UUIDType};
use shared::utils::{extract_extension_from_url, hash_bytes, hash_string_as_hex, truncate_string, ExportStyleConfig, CONSTANTS};
use std::collections::{HashMap, HashSet, VecDeque};
use std::path::{Path, PathBuf};
@@ -24,15 +20,6 @@ use std::sync::Arc;
use tokio::fs::{create_dir_all, remove_dir, remove_file, File};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
use serde::Deserialize;
use serde_json::value::RawValue as JsonRawValue;
#[derive(Deserialize)]
struct AdditionalProps<'a> {
#[serde(borrow)]
info: Option<&'a JsonRawValue>,
}
/// Sanitizes a string to be safe for use as a file or directory name by
/// following a strict "allow-list" approach and discarding invalid characters.
fn sanitize_for_filename(text: &str, underscore_whitespace: bool) -> String {
@@ -92,39 +79,6 @@ fn extract_match(name: &str, pattern: &Regex) -> (String, Option<String>) {
)
}
/// Extracts a number from a pattern (e.g., "S01" -> 1), falling back to a default value.
fn extract_season_or_episode_match_with_default(
name: &str,
pattern: &Regex,
default_value: Option<&String>,
) -> (String, Option<u32>) {
let (new_name, value) = extract_match(name, pattern);
let new_value = match value {
// we skip the prefix S or E to parse the num
Some(num_value) => num_value[1..].parse::<u32>().ok(),
None => default_value.and_then(|val| val.parse::<u32>().ok()),
};
(new_name, new_value)
}
// Extracts and removes an episode number (e.g., "E01") based on style patterns.
fn style_rename_season(
name: &str,
style: &ExportStyleConfig,
series_season: Option<&String>,
) -> (String, Option<u32>) {
extract_season_or_episode_match_with_default(name, &style.season, series_season)
}
/// Renames episode numbers in media titles according to style patterns
fn style_rename_episode(
name: &str,
style: &ExportStyleConfig,
series_episode: Option<&String>,
) -> (String, Option<u32>) {
extract_season_or_episode_match_with_default(name, &style.episode, series_episode)
}
/// Extracts and formats year information from media titles
fn style_rename_year<'a>(
name: &'a str,
@@ -168,91 +122,6 @@ fn style_rename_year<'a>(
(std::borrow::Cow::Owned(new_name), smallest_year)
}
#[derive(Clone)]
enum InputTmdbIndexTree {
Video(BPlusTree<u32, InputVodInfoRecord>),
Series(BPlusTree<u32, XtreamSeriesEpisode>),
}
#[derive(Clone)]
enum InputTmdbIndexValue {
Video(InputVodInfoRecord),
Series(XtreamSeriesEpisode),
}
type InputTmdbIndexMap = HashMap<String, Option<(FileReadGuard, InputTmdbIndexTree)>>;
async fn get_tmdb_value(
cfg: &AppConfig,
provider_id: Option<u32>,
input_name: &str,
input_indexes: &mut InputTmdbIndexMap,
item_type: PlaylistItemType,
) -> Option<InputTmdbIndexValue> {
// the tmdb_ids are stored inside record files for xtream input.
// we load this record files on request for each input and item_type.
let pid = provider_id?;
match input_indexes.entry(input_name.to_string()) {
std::collections::hash_map::Entry::Occupied(entry) => {
if let Some((_, tree_value)) = entry.get() {
match tree_value {
InputTmdbIndexTree::Video(tree) => tree
.query(&pid)
.map(|vod_record| InputTmdbIndexValue::Video(vod_record.clone())),
InputTmdbIndexTree::Series(tree) => tree
.query(&pid)
.map(|episode| InputTmdbIndexValue::Series(episode.clone())),
}
} else {
None
}
}
std::collections::hash_map::Entry::Vacant(entry) => {
if let Ok(Some(tmdb_path)) = get_input_storage_path(input_name, &cfg.config.load().working_dir)
.map(|storage_path| xtream_get_record_file_path(&storage_path, item_type))
{
{
let file_lock = cfg.file_locks.read_lock(&tmdb_path).await;
match item_type {
PlaylistItemType::Series
| PlaylistItemType::LocalSeries => {
if let Ok(tree) =
BPlusTree::<u32, XtreamSeriesEpisode>::load(&tmdb_path)
{
let tmdb_id = tree.query(&pid).map(|episode| {
InputTmdbIndexValue::Series(episode.clone())
});
entry.insert(Some((
file_lock,
InputTmdbIndexTree::Series(tree),
)));
return tmdb_id;
}
}
PlaylistItemType::Video
| PlaylistItemType::LocalVideo => {
if let Ok(tree) =
BPlusTree::<u32, InputVodInfoRecord>::load(&tmdb_path)
{
let tmdb_id = tree.query(&pid).map(|vod_record| {
InputTmdbIndexValue::Video(vod_record.clone())
});
entry.insert(Some((
file_lock,
InputTmdbIndexTree::Video(tree),
)));
return tmdb_id;
}
}
_ => {}
}
};
}
entry.insert(None);
None
}
}
}
pub fn strm_get_file_paths(file_prefix: &str, target_path: &Path) -> PathBuf {
target_path.join(PathBuf::from(format!("{file_prefix}_{}.{}", storage_const::FILE_STRM, storage_const::FILE_SUFFIX_DB)))
}
@@ -269,8 +138,8 @@ struct StrmItemInfo {
url: String,
series_name: Option<String>,
release_date: Option<String>,
season: Option<String>,
episode: Option<String>,
season: Option<u32>,
episode: Option<u32>,
added: Option<u64>,
tmdb_id: Option<u32>,
}
@@ -290,35 +159,34 @@ fn extract_item_info(pli: &mut PlaylistItem) -> StrmItemInfo {
let virtual_id = header.virtual_id;
let input_name = header.input_name.clone();
let url = header.url.clone();
let (series_name, release_date, added, season, episode, tmdb_id) = match header.item_type {
let (series_name, release_date, added, tmdb_id, season, episode) = match header.item_type {
PlaylistItemType::Series
| PlaylistItemType::LocalSeries => {
let series_name = match header.get_field("name") {
Some(name) if !name.is_empty() => Some(name.to_string()),
_ => header.get_additional_property_as_str("series_name"),
let series_name = Some(header.name.clone());
let (release_date, added, tmdb_id, season, episode) = match header.additional_properties.as_ref() {
None => (None, None, None, None, None),
Some(props) => (
props.get_release_date(),
props.get_added(),
props.get_tmdb_id().filter(|&id| id != 0),
props.get_season(),
props.get_episode(),
)
};
let release_date = header
.get_additional_property_as_str("series_release_date")
.or_else(|| header.get_additional_property_as_str("release_date"));
let season = header.get_additional_property_as_str("season");
let episode = header.get_additional_property_as_str("episode");
let added = header.get_additional_property_as_u64("added");
let tmdb_id = header
.get_additional_property_as_u32("tmdb_id")
.filter(|&id| id != 0)
.or_else(|| header.get_additional_property_as_u32("tmdb"));
(series_name, release_date, added, season, episode, tmdb_id)
(series_name, release_date, added, tmdb_id, season, episode)
}
PlaylistItemType::Video
| PlaylistItemType::LocalVideo => {
let name = header.get_field("name").map(|v| v.to_string());
let release_date = header.get_additional_property_as_str("release_date");
let added = header.get_additional_property_as_u64("added");
let tmdb_id = header
.get_additional_property_as_u32("tmdb_id")
.filter(|&id| id != 0)
.or_else(|| header.get_additional_property_as_u32("tmdb"));
(name, release_date, added, None, None, tmdb_id)
let name = Some(header.name.clone());
let (release_date, added, tmdb_id) = match header.additional_properties.as_ref() {
None => (None, None, None),
Some(props) => (
props.get_release_date(),
props.get_added(),
props.get_tmdb_id().filter(|&id| id != 0),
)
};
(name, release_date, added, tmdb_id, None, None)
}
_ => (None, None, None, None, None, None),
};
@@ -331,10 +199,10 @@ fn extract_item_info(pli: &mut PlaylistItem) -> StrmItemInfo {
input_name,
url,
series_name,
release_date,
release_date: release_date.as_deref().map(ToString::to_string),
season,
episode,
added,
added: added.as_ref().map_or_else(|| Some(0), |a| a.parse::<u64>().ok()),
tmdb_id,
}
}
@@ -478,10 +346,8 @@ fn format_for_kodi(
let series_folder_name = format!("{sanitized_series_name}{year_string}{id_string}");
let (_, season) = style_rename_season(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.season.as_ref());
let (_, episode) = style_rename_episode(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.episode.as_ref());
let season_num = season.unwrap_or(1);
let episode_num = episode.unwrap_or(1);
let season_num=strm_item_info.season.unwrap_or(1u32);
let episode_num = strm_item_info.episode.unwrap_or(1u32);
let final_filename = format!("{sanitized_series_name}{separator}S{season_num:02}E{episode_num:02}");
let season_folder = format!("Season{separator}{season_num:02}");
@@ -542,10 +408,8 @@ fn format_for_plex(
let series_folder_name = format!("{sanitized_series_name}{year_string}{id_string}");
let (_, season) = style_rename_season(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.season.as_ref());
let (_, episode) = style_rename_episode(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.episode.as_ref());
let season_num = season.unwrap_or(1);
let episode_num = episode.unwrap_or(1);
let season_num = strm_item_info.season.unwrap_or(1);
let episode_num = strm_item_info.episode.unwrap_or(1);
// Plex standard: lowercase 's' and hyphens as separators.
let final_filename = format!("{sanitized_series_name} - s{season_num:02}e{episode_num:02}");
@@ -609,10 +473,8 @@ fn format_for_emby(
let series_folder_name = format!("{sanitized_series_name}{year_string}{id_string}");
let (_, season) = style_rename_season(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.season.as_ref());
let (_, episode) = style_rename_episode(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.episode.as_ref());
let season_num = season.unwrap_or(1);
let episode_num = episode.unwrap_or(1);
let season_num = strm_item_info.season.unwrap_or(1);
let episode_num = strm_item_info.episode.unwrap_or(1);
// Emby/Jellyfin standard: uppercase 'S' and hyphens.
let final_filename = format!("{sanitized_series_name} - S{season_num:02}E{episode_num:02}");
@@ -674,10 +536,8 @@ fn format_for_jellyfin(
let series_folder_name = format!("{sanitized_series_name}{year_string}{id_string}");
let (_, season) = style_rename_season(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.season.as_ref());
let (_, episode) = style_rename_episode(&strm_item_info.title, &CONSTANTS.export_style_config, strm_item_info.episode.as_ref());
let season_num = season.unwrap_or(1);
let episode_num = episode.unwrap_or(1);
let season_num = strm_item_info.season.unwrap_or(1);
let episode_num = strm_item_info.episode.unwrap_or(1);
// Emby/Jellyfin standard: uppercase 'S' and hyphens.
let final_filename = format!("{sanitized_series_name} - S{season_num:02}E{episode_num:02}");
@@ -699,29 +559,17 @@ fn format_for_jellyfin(
/// Generates style-compliant directory and file names by dispatching
/// the call to a dedicated formatting function for the respective style.
async fn style_based_rename(
cfg: &AppConfig,
fn style_based_rename(
strm_item_info: &StrmItemInfo,
input_tmdb_indexes: &mut InputTmdbIndexMap,
style: &StrmExportStyle,
tmdb: Option<u32>,
style: StrmExportStyle,
underscore_whitespace: bool,
flat: bool,
) -> (PathBuf, String) {
let separator = if underscore_whitespace { "_" } else { " " };
let tmdb_id_val = get_tmdb_value(
cfg,
strm_item_info.provider_id,
strm_item_info.input_name.as_str(),
input_tmdb_indexes,
strm_item_info.item_type,
).await;
let tmdb_id = match tmdb_id_val {
Some(InputTmdbIndexValue::Video(r)) if r.tmdb_id != 0 => r.tmdb_id,
Some(InputTmdbIndexValue::Series(e)) if e.tmdb_id != 0 => e.tmdb_id,
_ => strm_item_info.tmdb_id.unwrap_or(0),
};
let tmdb_id = tmdb.or(strm_item_info.tmdb_id).unwrap_or(0);
// Dispatch the call to the responsible function based on the style.
match style {
@@ -732,8 +580,8 @@ async fn style_based_rename(
}
}
async fn prepare_strm_files(
cfg: &AppConfig,
fn prepare_strm_files(
_app_config: &AppConfig,
new_playlist: &mut [PlaylistGroup],
_root_path: &Path,
strm_target_output: &StrmTargetOutput,
@@ -746,7 +594,6 @@ async fn prepare_strm_files(
let mut all_filenames = HashSet::with_capacity(channel_count);
// contains only collision filenames
let mut collisions: HashSet<Arc<String>> = HashSet::new();
let mut input_tmdb_indexes: InputTmdbIndexMap = HashMap::with_capacity(channel_count);
let mut result = Vec::with_capacity(channel_count);
// first we create the names to identify name collisions
@@ -755,38 +602,16 @@ async fn prepare_strm_files(
let strm_item_info = extract_item_info(pli);
let (dir_path, strm_file_name) = style_based_rename(
cfg,
&strm_item_info,
&mut input_tmdb_indexes,
&strm_target_output.style,
pli.get_tmdb_id(),
strm_target_output.style,
strm_target_output.underscore_whitespace,
strm_target_output.flat,
).await;
);
// Conditionally generate the quality string based on the new config flag
let separator = if strm_target_output.underscore_whitespace { "_" } else { " " };
let quality_string = if strm_target_output.add_quality_to_filename {
pli.header.additional_properties
.as_ref()
.and_then(|raw| {
let props: AdditionalProps<'_> = serde_json::from_str(raw.get()).ok()?;
let info = props.info?;
serde_json::from_str::<serde_json::Value>(info.get()).ok()
})
.as_ref()
.and_then(MediaQuality::from_ffprobe_info)
.map_or_else(String::new, |quality| {
let formatted = quality.format_for_filename(separator);
if formatted.is_empty() {
String::new()
} else {
// Hard-coded separator for filename clarity.
format!(" - [{formatted}]")
}
})
} else {
String::new()
};
let quality_string = get_quality(strm_target_output, pli, separator);
let final_filename = format!("{strm_file_name}{quality_string}");
let filename = Arc::new(final_filename);
@@ -839,6 +664,32 @@ async fn prepare_strm_files(
result
}
fn get_quality(strm_target_output: &StrmTargetOutput, pli: &PlaylistItem, separator: &str) -> String {
if strm_target_output.add_quality_to_filename {
let (audio, video) = match pli.header.additional_properties.as_ref() {
None => (None, None),
Some(props) => {
match props {
StreamProperties::Live(_)
| StreamProperties::Series(_) => (None, None),
StreamProperties::Video(video) =>
video.details.as_ref().map_or_else(|| (None, None), |d| (d.audio.as_deref(), d.video.as_deref())),
StreamProperties::Episode(episode) =>
(episode.audio.as_deref(), episode.video.as_deref())
}
}
};
if let Some(media_quality) = MediaQuality::from_ffprobe_info(audio, video) {
let formatted = media_quality.format_for_filename(separator);
if !formatted.is_empty() {
// Hard-coded separator for filename clarity.
return format!(" - [{formatted}]")
}
}
}
String::new()
}
pub async fn write_strm_playlist(
app_config: &AppConfig,
target: &ConfigTarget,
@@ -850,7 +701,7 @@ pub async fn write_strm_playlist(
}
let config = app_config.config.load();
let Some(root_path) = utils::get_file_path(
let Some(root_path) = crate::utils::get_file_path(
&config.working_dir,
Some(std::path::PathBuf::from(&target_output.directory)),
) else {
@@ -860,7 +711,7 @@ pub async fn write_strm_playlist(
)));
};
let user_and_server_info = get_credentials_and_server_info(app_config, target_output.username.as_ref());
let user_and_server_info = get_credentials_and_server_info(app_config, target_output.username.as_deref());
let normalized_dir = normalize_string_path(&target_output.directory);
let strm_file_prefix = hash_string_as_hex(&normalized_dir);
let strm_index_path =
@@ -886,7 +737,7 @@ pub async fn write_strm_playlist(
new_playlist,
&root_path,
target_output,
).await;
);
for strm_file in strm_files {
// file paths
let output_path = truncate_filename(&root_path.join(&strm_file.dir_path), 255);
@@ -1044,7 +895,7 @@ async fn has_strm_file_same_hash(file_path: &PathBuf, content_hash: UUIDType) ->
fn get_credentials_and_server_info(
cfg: &AppConfig,
username: Option<&String>,
username: Option<&str>,
) -> Option<(ProxyUserCredentials, ApiProxyServerInfo)> {
let username = username?;
let credentials = cfg.get_user_credentials(username)?;
backend/src/repository/user_repository.rs
-1
View File
@@ -494,7 +494,6 @@ mod tests {
sources_file_path: String::new(),
mapping_file_path: None,
api_proxy_file_path: String::new(),
library_file_path: String::new(),
custom_stream_response_path: None,
}))),
file_locks: Arc::new(FileLockManager::default()),
backend/src/repository/xtream_playlist_iterator.rs
+21 -33
View File
@@ -1,25 +1,21 @@
use crate::model::XtreamMappingOptions;
use crate::model::{xtream_playlistitem_to_document, AppConfig, ProxyUserCredentials};
use crate::model::{xtream_mapping_option_from_target_options, AppConfig, ProxyUserCredentials};
use crate::model::ConfigTarget;
use crate::repository::indexed_document::IndexedDocumentIterator;
use crate::repository::bplustree::{BPlusTreeDiskIteratorOwned, BPlusTreeQuery};
use crate::repository::user_repository::user_get_bouquet_filter;
use crate::repository::xtream_repository::{xtream_get_file_paths, xtream_get_storage_path};
use crate::repository::xtream_repository::{xtream_get_file_path, xtream_get_storage_path};
use crate::utils::FileReadGuard;
use log::error;
use serde_json::Value;
use shared::error::info_err;
use shared::error::TuliproxError;
use shared::model::{PlaylistItemType, TargetType, XtreamCluster, XtreamPlaylistItem};
use shared::model::{PlaylistItemType, TargetType, XtreamCluster, XtreamMappingOptions, XtreamPlaylistItem};
use std::collections::HashSet;
pub struct XtreamPlaylistIterator {
reader: IndexedDocumentIterator<u32, XtreamPlaylistItem>,
reader: BPlusTreeDiskIteratorOwned<u32, XtreamPlaylistItem>,
options: XtreamMappingOptions,
cluster: XtreamCluster,
// Use parsed numeric filter to avoid per-item String allocations (no to_string per check)
filter_ids: Option<HashSet<u32>>,
base_url: String,
user: ProxyUserCredentials,
lookup_item: Option<(XtreamPlaylistItem, bool)>, // this is for filtered iteration
_file_lock: FileReadGuard,
}
@@ -38,17 +34,18 @@ impl XtreamPlaylistIterator {
let xtream_output = target.get_xtream_output().ok_or_else(|| info_err!(format!("Unexpected: xtream output required for target {}", target.name)))?;
let config = app_config.config.load();
if let Some(storage_path) = xtream_get_storage_path(&config, target.name.as_str()) {
let (xtream_path, idx_path) = xtream_get_file_paths(&storage_path, cluster);
if !xtream_path.exists() || !idx_path.exists() {
let xtream_path = xtream_get_file_path(&storage_path, cluster);
if !xtream_path.exists() {
return Err(info_err!(format!("No {cluster} entries found for target {}", &target.name)));
}
let file_lock = app_config.file_locks.read_lock(&xtream_path).await;
let reader = IndexedDocumentIterator::<u32, XtreamPlaylistItem>::new(&xtream_path, &idx_path)
.map_err(|err| info_err!(format!("Could not deserialize file {xtream_path:?} - {err}")))?;
let query = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&xtream_path)
.map_err(|err| info_err!(format!("Could not open BPlusTreeQuery {xtream_path:?} - {err}")))?;
let reader = query.disk_iter();
let options = XtreamMappingOptions::from_target_options(target, xtream_output, app_config);
let server_info = app_config.get_user_server_info(user);
let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str()));
let filter = user_get_bouquet_filter(&config, &user.username, category_id, TargetType::Xtream, cluster).await;
// Parse bouquet filter (strings) once into u32 set to minimize per-item allocations
@@ -67,8 +64,6 @@ impl XtreamPlaylistIterator {
cluster,
filter_ids,
_file_lock: file_lock,
base_url: server_info.get_base_url(),
user: user.clone(),
lookup_item: None,
})
} else {
@@ -77,11 +72,11 @@ impl XtreamPlaylistIterator {
}
fn matches_filters(cluster: XtreamCluster, filter_ids: Option<&HashSet<u32>>, item: &XtreamPlaylistItem) -> bool {
// We can't serve episodes within series
if cluster == XtreamCluster::Series
&& !matches!(item.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) {
return false;
}
return false;
}
// category_id-Filter
if let Some(set) = filter_ids {
@@ -94,28 +89,25 @@ impl XtreamPlaylistIterator {
}
fn get_next(&mut self) -> Option<(XtreamPlaylistItem, bool)> {
if self.reader.has_error() {
error!("Could not deserialize xtream item: {}", self.reader.get_path().display());
return None;
}
// reader no longer has manual error state, BPlusTreeQuery handles it via Result elsewhere
let filter_ids = self.filter_ids.as_ref();
let cluster = self.cluster;
let predicate = |(item, _): &(XtreamPlaylistItem, bool)| {
let predicate = |(_, item): &(u32, XtreamPlaylistItem)| {
Self::matches_filters(cluster, filter_ids, item)
};
if self.cluster == XtreamCluster::Series || self.filter_ids.is_some() {
if let Some((current_item, _)) = self.lookup_item.take() {
let next_valid = self.reader.find(predicate);
self.lookup_item = next_valid;
self.lookup_item = next_valid.map(|(_, item)| (item, true));
let has_next = self.lookup_item.is_some();
Some((current_item, has_next))
} else {
let current_item = self.reader.find(predicate);
if let Some((item, _)) = current_item {
self.lookup_item = self.reader.find(predicate);
if let Some((_, item)) = current_item {
self.lookup_item = self.reader.find(predicate).map(|(_, item)| (item, true));
let has_next = self.lookup_item.is_some();
Some((item, has_next))
} else {
@@ -123,7 +115,7 @@ impl XtreamPlaylistIterator {
}
}
} else {
self.reader.next()
self.reader.next().map(|(_, item)| (item, !self.reader.is_empty()))
}
}
}
@@ -154,14 +146,10 @@ impl XtreamPlaylistJsonIterator {
}
}
pub fn to_doc(pli: &XtreamPlaylistItem, url: &str, options: &XtreamMappingOptions, user: &ProxyUserCredentials) -> Value {
xtream_playlistitem_to_document(pli, url, options, user)
}
impl Iterator for XtreamPlaylistJsonIterator {
type Item = (String, bool);
fn next(&mut self) -> Option<Self::Item> {
self.inner.get_next().map(|(pli, has_next)| (to_doc(&pli, &self.inner.base_url, &self.inner.options, &self.inner.user).to_string(), has_next))
self.inner.get_next().map(|(pli, has_next)| (pli.to_document(&self.inner.options).to_string(), has_next))
}
}
backend/src/repository/xtream_repository.rs
+356 -771
View File
File diff suppressed because it is too large Load Diff
backend/src/tools/lru_cache.rs
+50 -24
View File
@@ -1,11 +1,16 @@
use crate::utils::{traverse_dir};
use shared::utils::{hash_string_as_hex, human_readable_byte_size, sanitize_sensitive_info};
use crate::utils::{decode_base64_string, encode_base64_hash, encode_base64_string, traverse_dir};
use shared::utils::{human_readable_byte_size, sanitize_sensitive_info};
use log::{debug, error, info, trace};
use std::collections::{HashMap, VecDeque};
use std::fs;
use std::path::PathBuf;
use crate::utils::{trace_if_enabled};
#[inline]
fn encode_cache_key(key: &str) -> String {
encode_base64_hash(key)
}
/// `LRUResourceCache`
///
/// A least-recently-used (LRU) file-based resource cache that stores files in a directory on disk,
@@ -22,7 +27,7 @@ pub struct LRUResourceCache {
capacity: usize, // Maximum size in bytes
cache_dir: PathBuf,
current_size: usize, // Current size in bytes
cache: HashMap<String, (PathBuf, usize)>,
cache: HashMap<String, (PathBuf, Option<String>, usize)>,
usage_order: VecDeque<String>,
}
@@ -33,11 +38,13 @@ impl LRUResourceCache {
/// - `cache_dir`: The directory path where cached files are stored.
///
pub fn new(capacity: usize, cache_dir: &str) -> Self {
// Estimate: assume average file size of 256KB
let estimated_entries = (capacity / (256 * 1024)).clamp(64, 16384);
Self {
capacity,
cache_dir: PathBuf::from(cache_dir),
current_size: 0,
cache: HashMap::<String, (PathBuf, usize)>::new(),
cache: HashMap::<String, (PathBuf, Option<String>, usize)>::with_capacity(estimated_entries),
usage_order: VecDeque::new(),
}
}
@@ -53,15 +60,22 @@ impl LRUResourceCache {
pub fn scan(&mut self) -> std::io::Result<()> {
let mut visit = |entry: &std::fs::DirEntry, metadata: &std::fs::Metadata| {
let path = entry.path();
if let Some(file_name) = path.file_name() {
let key = String::from(file_name.to_string_lossy());
if let Some(os_file_name) = path.file_name() {
let file_name = String::from(os_file_name.to_string_lossy());
let (key, mime_type) = if let Some((part1, part2)) = file_name.split_once('.') {
(part1.to_string(), String::from_utf8(decode_base64_string(part2)).ok())
} else {
(file_name.clone(), None)
};
let file_size = usize::try_from(metadata.len()).unwrap_or(0);
// we need to duplicate because of closure we cant call insert_to_cache
{ // insert_to_cache
let mut path = self.cache_dir.clone();
path.push(&key);
path.push(&file_name);
trace!("Added file to cache: {}", &path.to_string_lossy());
self.cache.insert(key.clone(), (path.clone(), file_size));
self.cache.insert(key.clone(), (path.clone(), mime_type, file_size));
self.usage_order.push_back(key);
self.current_size += file_size;
}
@@ -83,55 +97,67 @@ impl LRUResourceCache {
/// - `file_size`: The size of the file in bytes.
/// - Returns:
/// - The `PathBuf` where the file is stored.
pub fn add_content(&mut self, url: &str, file_size: usize) -> std::io::Result<PathBuf> {
let key = hash_string_as_hex(url);
let path = self.insert_to_cache(key, file_size);
pub fn add_content(&mut self, url: &str, mime_type: Option<String>, file_size: usize) -> std::io::Result<PathBuf> {
let key = encode_cache_key(url);
let path = self.insert_to_cache(key, mime_type, file_size);
if self.current_size > self.capacity {
self.evict_if_needed();
}
Ok(path)
}
fn insert_to_cache(&mut self, key: String, file_size: usize) -> PathBuf {
let mut path = self.cache_dir.clone();
path.push(&key);
fn insert_to_cache(&mut self, key: String, mime_type: Option<String>, file_size: usize) -> PathBuf {
let path = self.get_store_path(&key, mime_type.as_deref());
debug!("Added file to cache: {}", &path.to_string_lossy());
self.cache.insert(key.clone(), (path.clone(), file_size));
self.cache.insert(key.clone(), (path.clone(), mime_type, file_size));
self.usage_order.push_back(key);
self.current_size += file_size;
path
}
pub fn store_path(&self, url: &str) -> PathBuf {
let key = hash_string_as_hex(url);
pub fn store_path(&self, url: &str, mime_type: Option<&str>) -> PathBuf {
self.get_store_path(&encode_cache_key(url), mime_type)
}
fn get_store_path(&self, cache_key: &str, mime_type: Option<&str>) -> PathBuf {
let key = if let Some(mime) = mime_type {
format!("{cache_key}.{}", encode_base64_string(mime.as_bytes()))
} else {
cache_key.to_string()
};
let mut path = self.cache_dir.clone();
path.push(&key);
path
}
/// - Retrieves a file from the cache if it exists.
/// - Moves the file's key to the end of the usage queue to mark it as recently used.
/// - Arguments:
/// - `url`: The unique identifier for the file.
/// - Returns:
/// - The `PathBuf` of the file if it exists; `None` otherwise.
pub fn get_content(&mut self, url: &str) -> Option<PathBuf> {
let key = hash_string_as_hex(url);
pub fn get_content(&mut self, url: &str) -> Option<(PathBuf, Option<String>)> {
let key = encode_cache_key(url);
{
if let Some((path, size)) = self.cache.get(&key) {
if let Some((path, mime_type, size)) = self.cache.get(&key) {
if path.exists() {
trace_if_enabled!("Responding resource from cache with key: {key} for url: {}", sanitize_sensitive_info(url));
// Move to the end of the queue
self.usage_order.retain(|k| k != &key); // remove from queue
if let Some(pos) = self.usage_order.iter().position(|k| k == &key) {
self.usage_order.remove(pos); // remove from queue
}
self.usage_order.push_back(key); // add to the to end
return Some(path.clone());
return Some((path.clone(), mime_type.clone()));
}
{
trace_if_enabled!("Cache inconsistency: file missing for key: {key}, url: {}", sanitize_sensitive_info(url));
// this should not happen, someone deleted the file manually and the cache is not in sync
self.current_size -= size;
self.cache.remove(&key);
self.usage_order.retain(|k| k != &key);
if let Some(pos) = self.usage_order.iter().position(|k| k == &key) {
self.usage_order.remove(pos);
}
}
}
}
@@ -142,7 +168,7 @@ impl LRUResourceCache {
// if the cache size is to small and one element exceeds the size than the cache won't work, we ignore this
while self.current_size > self.capacity {
if let Some(oldest_file) = self.usage_order.pop_front() {
if let Some((file, size)) = self.cache.remove(&oldest_file) {
if let Some((file, _mime_type, size)) = self.cache.remove(&oldest_file) {
self.current_size -= size;
if let Err(err) = fs::remove_file(&file) {
error!("Failed to delete cached file {} {err}", file.to_string_lossy());
backend/src/utils/binary_utils.rs
+22
View File
@@ -0,0 +1,22 @@
use std::io;
use log::error;
use shared::error::to_io_error;
#[inline]
pub fn binary_serialize<T>(value: &T) -> io::Result<Vec<u8>>
where
T: ?Sized + serde::Serialize,
{
rmp_serde::to_vec(value).map_err(to_io_error)
}
#[inline]
pub fn binary_deserialize<T>(value: &[u8]) -> io::Result<T>
where
T: for<'a> serde::Deserialize<'a>,
{
rmp_serde::from_slice(value).map_err(|e| {
error!("Failed to decode {e}");
to_io_error(e)
})
}
backend/src/utils/bincode_utils.rs
-25
View File
@@ -1,25 +0,0 @@
use std::io;
use log::error;
use shared::error::to_io_error;
#[inline]
pub fn bincode_serialize<T>(value: &T) -> io::Result<Vec<u8>>
where
T: ?Sized + serde::Serialize,
{
bincode::serde::encode_to_vec(value, bincode::config::legacy()).map_err(to_io_error)
}
#[inline]
pub fn bincode_deserialize<T>(value: &[u8]) -> io::Result<T>
where
T: for<'a> serde::Deserialize<'a>,
{
match bincode::serde::decode_from_slice(value, bincode::config::legacy()) {
Ok((instance, _size)) => Ok(instance),
Err(e) => {
error!("Failed to decode {e}");
Err(to_io_error(e))
},
}
}
backend/src/utils/crypto_utils.rs
+7 -2
View File
@@ -3,11 +3,16 @@ use openssl::symm::{Cipher, Crypter, Mode};
use rand::{RngCore, rngs::OsRng, TryRngCore};
use shared::error::{TuliproxError, TuliproxErrorKind};
fn encode_base64_string(input: &[u8]) -> String {
pub fn encode_base64_hash(text: &str) -> String {
let hash = blake3::hash(text.as_bytes());
encode_base64_string(hash.as_bytes())
}
pub fn encode_base64_string(input: &[u8]) -> String {
general_purpose::URL_SAFE_NO_PAD.encode(input)
}
fn decode_base64_string(input: &str) -> Vec<u8> {
pub fn decode_base64_string(input: &str) -> Vec<u8> {
general_purpose::URL_SAFE_NO_PAD.decode(input).unwrap_or_else(|_| input.as_bytes().to_vec())
}
backend/src/utils/db_viewer.rs
+89
View File
@@ -0,0 +1,89 @@
use crate::repository::bplustree::{BPlusTreeDiskIterator, BPlusTreeQuery};
use env_logger::{Builder, Target};
use log::{error, LevelFilter};
use serde::{Deserialize, Serialize};
use shared::model::{M3uPlaylistItem, XtreamPlaylistItem};
use std::io::Write;
use std::path::{PathBuf};
#[derive(Debug, Copy, Clone)]
enum DbType {
Xtream,
M3u,
}
pub fn db_viewer(xtream_filename: Option<&str>, m3u_filename: Option<&str>) {
let mut any_processed = false;
if let Some(filename) = xtream_filename {
any_processed |= dump_db(filename, DbType::Xtream);
}
if let Some(filename) = m3u_filename {
any_processed |= dump_db(filename, DbType::M3u);
}
exit_app(if any_processed { 0 } else { 1 });
}
fn dump_db(filename: &str, db_type: DbType) -> bool {
let mut log_builder = Builder::from_default_env();
log_builder.target(Target::Stderr);
log_builder.filter_level(LevelFilter::Info);
let _ = log_builder.try_init();
match PathBuf::from(filename).canonicalize() {
Ok(path) => {
match db_type {
DbType::Xtream => {
if let Ok(mut query) = BPlusTreeQuery::<u32, XtreamPlaylistItem>::try_new(&path) {
let iterator = query.iter();
return print_json_from_iter(iterator);
}
}
DbType::M3u => {
if let Ok(mut query) = BPlusTreeQuery::<u32, M3uPlaylistItem>::try_new(&path) {
let iterator = query.iter();
return print_json_from_iter(iterator);
}
}
}
}
Err(err) => {
error!("Invalid file path! {err}");
}
}
return false;
}
fn print_json_from_iter<P>(iterator: BPlusTreeDiskIterator<u32, P>) -> bool
where
P: Serialize + for<'de> Deserialize<'de> + Clone,
{
let mut error_count = 0;
println!("[");
let mut first = true;
for (_, entry) in iterator {
match serde_json::to_string(&entry) {
Ok(text) => {
if !first {
println!(",");
}
println!("{text}");
first = false;
}
Err(err) => {
error!("Failed: {err}");
error_count += 1;
},
}
}
println!("]");
if error_count > 0 { false } else { true }
}
fn exit_app(code: i32) {
let _ = std::io::stdout().flush();
let _ = std::io::stderr().flush();
std::process::exit(code);
}
backend/src/utils/file/file_utils.rs
+19 -3
View File
@@ -351,9 +351,25 @@ pub fn cleanup_unlisted_files_with_suffix(
for entry in (fs::read_dir(dir)?).flatten() {
let path = entry.path();
if path.is_file() && !keep_set.contains(&path) && path.file_name()
.and_then(|n| n.to_str())
.is_some_and(|name| name.ends_with(suffix)) && fs::remove_file(&path).is_ok() {
if !path.is_file() || keep_set.contains(&path) {
continue;
}
let delete = {
let zero_size = entry
.metadata()
.map(|m| m.len() == 0)
.unwrap_or(false);
let suffix_match = path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|name| name.ends_with(suffix));
zero_size || suffix_match
};
if delete && fs::remove_file(&path).is_ok() {
debug!("Deleted {:?}", path.display());
}
}
backend/src/utils/geoip.rs
-2
View File
@@ -1,5 +1,4 @@
use crate::repository::bplustree::BPlusTree;
use serde::{Deserialize, Serialize};
use std::io;
use std::io::BufRead;
use std::net::Ipv4Addr;
@@ -9,7 +8,6 @@ fn ipv4_to_u32(ip: &str) -> Option<u32> {
ip.parse::<Ipv4Addr>().ok().map(u32::from)
}
#[derive(Serialize, Deserialize)]
pub struct GeoIp {
tree: BPlusTree<u32, (u32, String)>,
}
backend/src/utils/logging.rs
+2 -2
View File
@@ -23,7 +23,7 @@ fn get_log_level(log_level: &str) -> LevelFilter {
}
}
pub fn init_logger(user_log_level: Option<&String>, config_file: &str) {
pub fn init_logger(user_log_level: Option<&str>, config_file: &str) {
// tracing_subscriber::registry()
@@ -74,6 +74,6 @@ pub fn init_logger(user_log_level: Option<&String>, config_file: &str) {
for module in LOG_ERROR_LEVEL_MOD {
log_builder.filter_module(module, LevelFilter::Error);
}
log_builder.init();
let _ = log_builder.try_init();
info!("Log Level {}", &log_levels.join(", "));
}
backend/src/utils/mod.rs
+4 -2
View File
@@ -7,15 +7,17 @@ mod step_measure;
mod logging;
mod trakt;
mod json_utils;
mod bincode_utils;
mod binary_utils;
mod telegram;
mod geoip;
mod db_viewer;
pub use self::bincode_utils::*;
pub use self::binary_utils::*;
pub use self::logging::*;
pub use self::trakt::*;
pub use self::telegram::*;
pub use self::geoip::*;
pub use self::db_viewer::*;
#[macro_export]
macro_rules! debug_if_enabled {
backend/src/utils/network/m3u.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use crate::processing::parser::m3u;
use crate::utils::prepare_file_path;
use crate::utils::request;
pub async fn get_m3u_playlist(client: &reqwest::Client, cfg: &Arc<Config>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
pub async fn download_m3u_playlist(client: &reqwest::Client, cfg: &Arc<Config>, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let working_dir = &cfg.working_dir;
let input_source: InputSource = {
match input.staged.as_ref() {
backend/src/utils/network/request.rs
+13 -8
View File
@@ -61,7 +61,7 @@ pub fn content_type_from_ext(ext: &str) -> &'static str {
"avi" => "video/x-msvideo",
"mov" => "video/quicktime",
"webm" => "video/webm",
"ts" => "video/mp2t",
"ts" => "video/mp2t",
_ => "application/octet-stream",
}
}
@@ -108,7 +108,6 @@ pub async fn get_input_epg_content_as_file(client: &reqwest::Client, input: &Con
}
}
pub async fn get_input_text_content(client: &reqwest::Client, input: &InputSource, working_dir: &str, persist_filepath: Option<PathBuf>) -> Result<String, TuliproxError> {
debug_if_enabled!("getting input text content working_dir: {}, url: {}", working_dir, sanitize_sensitive_info(&input.url));
@@ -415,12 +414,9 @@ pub async fn get_remote_content_as_stream(
}
async fn get_remote_content(client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) -> Result<(String, String), Error> {
let start_time = Instant::now();
let (mut stream, response_url) = get_remote_content_as_stream(client, input, headers, url, disabled_headers).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
let mut content = String::new();
stream.read_to_string(&mut content).await.map_err(|e| str_to_io_error(&format!("Failed to read content: {e}")))?;
debug_if_enabled!("Request took: {} {}", format_elapsed_time(start_time.elapsed().as_secs()), sanitize_sensitive_info(url.as_str()));
Ok((content, response_url))
}
@@ -456,7 +452,8 @@ pub async fn download_text_content(
headers: Option<&HeaderMap>,
persist_filepath: Option<PathBuf>,
) -> Result<(String, String), Error> {
if let Ok(url) = input.url.parse::<url::Url>() {
let start_time = Instant::now();
let result = if let Ok(url) = input.url.parse::<url::Url>() {
let result = if url.scheme() == "file" {
match url.to_file_path() {
Ok(file_path) => get_local_file_content(&file_path).await.map(|c| (c, url.to_string())),
@@ -479,7 +476,15 @@ pub async fn download_text_content(
}
} else {
Err(str_to_io_error(&format!("Malformed URL {}", sanitize_sensitive_info(&input.url))))
};
if log_enabled!(log::Level::Debug) {
if let Ok((_content, response_url)) = result.as_ref() {
debug!("Request took: {} {}", format_elapsed_time(start_time.elapsed().as_secs()), sanitize_sensitive_info(response_url.as_str()));
}
}
result
}
pub async fn download_text_content_as_stream(
@@ -520,7 +525,7 @@ pub async fn download_text_content_as_stream(
}
async fn download_json_content(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<serde_json::Value, Error> {
debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url));
debug_if_enabled!("Downloading json content from {}", sanitize_sensitive_info(&input.url));
match download_text_content(client, disabled_headers, input, None, persist_filepath).await {
Ok((content, _response_url)) => {
match serde_json::from_str::<serde_json::Value>(&content) {
@@ -540,7 +545,7 @@ pub async fn get_input_json_content(client: &reqwest::Client, disabled_headers:
}
async fn download_json_content_as_stream(client: &reqwest::Client, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, input: &InputSource, persist_filepath: Option<PathBuf>) -> Result<DynReader, Error> {
debug_if_enabled!("downloading json content from {}", sanitize_sensitive_info(&input.url));
debug_if_enabled!("Downloading json content from {}", sanitize_sensitive_info(&input.url));
match download_text_content_as_stream(client, disabled_headers, input, None, persist_filepath).await {
Ok((reader, _response_url)) => Ok(reader),
Err(err) => Err(err)
backend/src/utils/network/xtream.rs
+198 -64
View File
@@ -1,16 +1,22 @@
use crate::api::model::AppState;
use crate::messaging::send_message;
use crate::model::{is_input_expired, Config, ConfigInput, ConfigTarget, XtreamLoginInfo};
use crate::model::{is_input_expired, xtream_mapping_option_from_target_options, Config, ConfigInput, ConfigTarget, XtreamLoginInfo, XtreamTargetOutput};
use crate::model::{InputSource, ProxyUserCredentials};
use crate::processing::parser::xtream;
use crate::repository::xtream_repository;
use crate::repository::xtream_repository::{rewrite_xtream_series_info_content, rewrite_xtream_vod_info_content, xtream_get_input_info};
use crate::processing::parser::xtream::parse_xtream_series_info;
use crate::repository::playlist_repository::{get_target_id_mapping, rewrite_provider_series_info_episode_virtual_id, ProviderEpisodeKey};
use crate::repository::storage::{get_input_storage_path, get_target_storage_path};
use crate::repository::target_id_mapping::VirtualIdRecord;
use crate::repository::xtream_repository::{persist_input_vod_info, persists_input_series_info, write_playlist_batch_item_upsert, write_playlist_item_update};
use crate::utils::request;
use chrono::{DateTime, Utc};
use log::{error, info, warn};
use shared::error::{str_to_io_error, TuliproxError};
use shared::model::{MsgKind, PlaylistEntry, PlaylistGroup, ProxyUserStatus, XtreamCluster, XtreamPlaylistItem};
use shared::utils::{extract_extension_from_url, get_i64_from_serde_value, get_string_from_serde_value};
use shared::error::{str_to_io_error, to_io_error, TuliproxError};
use shared::model::{MsgKind, PlaylistEntry, PlaylistGroup, ProxyUserStatus, SeriesStreamProperties,
StreamProperties, VideoStreamProperties, XtreamCluster,
XtreamPlaylistItem, XtreamSeriesInfo, XtreamVideoInfo};
use shared::utils::{extract_extension_from_url, get_i64_from_serde_value, get_string_from_serde_value, sanitize_sensitive_info};
use std::collections::HashMap;
use std::io::Error;
use std::str::FromStr;
use std::sync::Arc;
@@ -53,64 +59,176 @@ pub async fn get_xtream_stream_info_content(client: &reqwest::Client, input: &In
}
}
#[allow(clippy::too_many_arguments)]
pub async fn get_xtream_stream_info<P>(client: &reqwest::Client,
app_state: &Arc<AppState>,
user: &ProxyUserCredentials,
input: &ConfigInput,
target: &ConfigTarget,
pli: &P,
info_url: &str,
cluster: XtreamCluster) -> Result<String, Error>
where
P: PlaylistEntry,
{
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
pub async fn get_xtream_stream_info(client: &reqwest::Client,
app_state: &Arc<AppState>,
user: &ProxyUserCredentials,
input: &ConfigInput,
target: &ConfigTarget,
pli: &XtreamPlaylistItem,
info_url: &str,
cluster: XtreamCluster) -> Result<String, Error> {
let xtream_output = target.get_xtream_output().ok_or_else(|| Error::other("Unexpected error, missing xtream output"))?;
let app_config = &app_state.app_config;
let server_info = app_config.get_user_server_info(user);
let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str()));
if cluster == XtreamCluster::Series {
if let Some(content) = xtream_repository::xtream_load_series_info(app_config, target.name.as_str(), pli.get_virtual_id()).await {
// Deliver existing target content
return rewrite_xtream_series_info_content(app_state, target, xtream_output, pli, user, &content).await;
}
// Check if the content has been resolved
if xtream_output.resolve_series {
if let Some(provider_id) = pli.get_provider_id() {
if let Some(content) = xtream_get_input_info(app_config, input, provider_id, XtreamCluster::Series).await {
return xtream_repository::write_and_get_xtream_series_info(app_state, target, xtream_output, pli, user, &content).await;
}
}
}
} else if cluster == XtreamCluster::Video {
if let Some(content) = xtream_repository::xtream_load_vod_info(app_config, target.name.as_str(), pli.get_virtual_id()).await {
// Deliver existing target content
return rewrite_xtream_vod_info_content(app_config, target, xtream_output, pli, user, &content);
}
// Check if the content has been resolved
if xtream_output.resolve_vod {
if let Some(provider_id) = pli.get_provider_id() {
if let Some(content) = xtream_get_input_info(app_config, input, provider_id, XtreamCluster::Video).await {
return xtream_repository::write_and_get_xtream_vod_info(app_config, target, xtream_output, pli, user, &content).await;
}
}
}
if let Some(content) = pli.get_resolved_info_document(&options) {
return serde_json::to_string(&content).map_err(to_io_error);
}
let input_source = InputSource::from(input).with_url(info_url.to_owned());
if let Ok(content) = get_xtream_stream_info_content(client, &input_source).await {
return match cluster {
XtreamCluster::Live => Ok(content),
XtreamCluster::Video => xtream_repository::write_and_get_xtream_vod_info(app_config, target, xtream_output, pli, user, &content).await,
XtreamCluster::Series => xtream_repository::write_and_get_xtream_series_info(app_state, target, xtream_output, pli, user, &content).await,
};
if let Some(provider_id) = pli.get_provider_id() {
match cluster {
XtreamCluster::Live => {}
XtreamCluster::Video => {
let working_dir = &app_config.config.load().working_dir;
if let Ok(storage_path) = get_input_storage_path(&input.name, working_dir) {
match serde_json::from_str::<XtreamVideoInfo>(&content) {
Ok(info) => {
// parse downloaded info into StreamProperties
let video_stream_props = VideoStreamProperties::from_info(&info, pli);
// persist input info
if let Err(err) = persist_input_vod_info(&app_state.app_config, &storage_path, cluster, &input.name, provider_id, &video_stream_props).await {
error!("Failed to persist video stream for input {}: {err}", &input.name);
}
// update target playlist
let mut vod_pli = pli.clone();
vod_pli.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props)));
if let Err(err) = write_playlist_item_update(app_config, &target.name, &vod_pli).await {
error!("Failed to persist video stream: {err}");
}
if target.use_memory_cache {
app_state.playlists.update_playlist_items(target, vec![&vod_pli]).await;
}
if let Some(value) = xtream_resolve_stream_info(app_state, user, target, xtream_output, &vod_pli) {
return value;
}
}
Err(err) => error!("Failed to persist video info: {err}")
}
}
}
XtreamCluster::Series => {
let working_dir = &app_config.config.load().working_dir;
let group = pli.get_group();
let series_name = pli.get_name();
match serde_json::from_str::<XtreamSeriesInfo>(&content) {
Ok(info) => {
// parse series info
let series_stream_props = SeriesStreamProperties::from_info(&info, pli);
if let Ok(storage_path) = get_input_storage_path(&input.name, working_dir) {
// update input db
if let Err(err) = persists_input_series_info(app_config, &storage_path, cluster, &input.name, provider_id, &series_stream_props).await {
error!("Failed to persist series info for input {}: {err}", &input.name);
}
}
if let Some(mut episodes) = parse_xtream_series_info(&pli.get_uuid(), &series_stream_props, &group, &series_name, input) {
let config = &app_state.app_config.config.load();
match get_target_storage_path(config, target.name.as_str()) {
None => {
error!("Failed to get target storage path {}. Cant save episodes", &target.name);
}
Some(target_path) => {
let mut in_memory_updates = Vec::new();
let mut provider_series: HashMap<String, Vec<ProviderEpisodeKey>> = HashMap::new();
{
let (mut target_id_mapping, _file_lock) = get_target_id_mapping(&app_state.app_config, &target_path).await;
if let Some(parent_id) = pli.get_provider_id() {
let category_id = pli.get_category_id().unwrap_or(0);
for episode in &mut episodes {
episode.header.virtual_id = target_id_mapping.get_and_update_virtual_id(&episode.header.uuid, provider_id, episode.header.item_type, parent_id);
episode.header.category_id = category_id;
let episode_provider_id = episode.header.get_provider_id().unwrap_or(0);
provider_series.entry(pli.get_uuid().to_string())
.or_default()
.push(ProviderEpisodeKey {
provider_id: episode_provider_id,
virtual_id: episode.header.virtual_id,
});
if target.use_memory_cache {
in_memory_updates.push(
VirtualIdRecord::new(
episode.header.get_provider_id().unwrap_or(0),
episode.header.virtual_id,
episode.header.item_type,
provider_id,
episode.get_uuid(),
),
);
}
}
}
if let Err(err) = target_id_mapping.persist() {
error!("Failed to persist target id mapping: {err}");
}
}
let xtream_episodes: Vec<XtreamPlaylistItem> = episodes.iter().map(XtreamPlaylistItem::from).collect();
if let Err(err) = write_playlist_batch_item_upsert(
app_config,
&target.name,
XtreamCluster::Series,
&xtream_episodes).await {
error!("Failed to persist playlist batch item update: {err}");
}
if target.use_memory_cache && !in_memory_updates.is_empty() {
app_state.playlists.insert_playlist_items(target, episodes).await;
app_state.playlists.update_target_id_mapping(target, in_memory_updates).await;
}
if !provider_series.is_empty() {
let mut series_pli = pli.clone();
series_pli.additional_properties = Some(StreamProperties::Series(Box::new(series_stream_props)));
rewrite_provider_series_info_episode_virtual_id(&mut series_pli, &provider_series);
if let Err(err) = write_playlist_item_update(app_config, &target.name, &series_pli).await {
error!("Failed to persist series stream: {err}");
}
app_state.playlists.update_playlist_items(target, vec![&series_pli]).await;
if let Some(value) = xtream_resolve_stream_info(app_state, user, target, xtream_output, &series_pli) {
return value;
}
}
}
}
}
}
Err(err) => {
error!("Failed to persist series info: {err}");
}
}
}
}
}
}
Err(str_to_io_error(&format!("Cant find stream with id: {}/{}/{}",
target.name.replace(' ', "_").as_str(), &cluster, pli.get_virtual_id())))
}
fn xtream_resolve_stream_info(app_state: &Arc<AppState>, user: &ProxyUserCredentials,
target: &ConfigTarget, xtream_output: &XtreamTargetOutput,
pli: &XtreamPlaylistItem) -> Option<Result<String, Error>> {
let app_config = &app_state.app_config;
let server_info = app_config.get_user_server_info(user);
let options = xtream_mapping_option_from_target_options(target, xtream_output, app_config, user, Some(server_info.get_base_url().as_str()));
if let Some(content) = pli.get_resolved_info_document(&options) {
return Some(serde_json::to_string(&content).map_err(to_io_error));
}
None
}
fn get_skip_cluster(input: &ConfigInput) -> Vec<XtreamCluster> {
let mut skip_cluster = vec![];
if let Some(input_options) = &input.options {
@@ -139,7 +257,8 @@ async fn xtream_login(cfg: &Config, client: &reqwest::Client, input: &InputSourc
let content = if let Ok(content) = request::get_input_json_content(client, None, input, None).await {
content
} else {
let input_source_account_info = input.with_url(format!("{}&action={}", &input.url, crate::model::XC_ACTION_GET_ACCOUNT_INFO));
let input_source_account_info =
input.with_url(format!("{}&action={}", &input.url, crate::model::XC_ACTION_GET_ACCOUNT_INFO));
match request::get_input_json_content(client, None, &input_source_account_info, None).await {
Ok(content) => content,
Err(err) => {
@@ -161,7 +280,8 @@ async fn xtream_login(cfg: &Config, client: &reqwest::Client, input: &InputSourc
login_info.status = Some(cur_status);
if !matches!(cur_status, ProxyUserStatus::Active | ProxyUserStatus::Trial) {
warn!("User status for user {username} is {cur_status:?}");
send_message(client, MsgKind::Info, cfg.messaging.as_ref(), &format!("User status for user {username} is {cur_status:?}")).await;
send_message(client, MsgKind::Info, cfg.messaging.as_ref(),
&format!("User status for user {username} is {cur_status:?}")).await;
}
}
}
@@ -182,7 +302,8 @@ async fn xtream_login(cfg: &Config, client: &reqwest::Client, input: &InputSourc
}
}
pub async fn notify_account_expire(exp_date: Option<i64>, cfg: &Config, client: &reqwest::Client, username: &str, input_name: &str) {
pub async fn notify_account_expire(exp_date: Option<i64>, cfg: &Config, client: &reqwest::Client,
username: &str, input_name: &str) {
if let Some(expiration_timestamp) = exp_date {
let now_secs = Utc::now().timestamp(); // UTC-Time
if expiration_timestamp > now_secs {
@@ -192,17 +313,20 @@ pub async fn notify_account_expire(exp_date: Option<i64>, cfg: &Config, client:
if let Some(datetime) = DateTime::<Utc>::from_timestamp(expiration_timestamp, 0) {
let formatted = datetime.format("%Y-%m-%d %H:%M:%S").to_string();
warn!("User account for user {username} expires {formatted}");
send_message(client, MsgKind::Info, cfg.messaging.as_ref(), &format!("User account for user {username} expires {formatted}")).await;
send_message(client, MsgKind::Info, cfg.messaging.as_ref(),
&format!("User account for user {username} expires {formatted}")).await;
}
}
} else {
warn!("User account for user {username} is expired");
send_message(client, MsgKind::Info, cfg.messaging.as_ref(), &format!("User account for user {username} for provider {input_name} is expired")).await;
send_message(client, MsgKind::Info, cfg.messaging.as_ref(),
&format!("User account for user {username} for provider {input_name} is expired")).await;
}
}
}
pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &reqwest::Client, input: &Arc<ConfigInput>) -> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
pub async fn download_xtream_playlist(cfg: &Arc<Config>, client: &reqwest::Client, input: &Arc<ConfigInput>)
-> (Vec<PlaylistGroup>, Vec<TuliproxError>) {
let input_source: InputSource = {
match input.staged.as_ref() {
None => input.as_ref().into(),
@@ -233,8 +357,10 @@ pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &reqwest::Client, in
if !skip_cluster.contains(xtream_cluster) {
let input_source_category = input_source.with_url(format!("{base_url}&action={category}"));
let input_source_stream = input_source.with_url(format!("{base_url}&action={stream}"));
let category_file_path = crate::utils::prepare_file_path(input.persist.as_deref(), working_dir, format!("{category}_").as_str());
let stream_file_path = crate::utils::prepare_file_path(input.persist.as_deref(), working_dir, format!("{stream}_").as_str());
let category_file_path = crate::utils::prepare_file_path(input.persist.as_deref(),
working_dir, format!("{category}_").as_str());
let stream_file_path = crate::utils::prepare_file_path(input.persist.as_deref(),
working_dir, format!("{stream}_").as_str());
match futures::join!(
request::get_input_json_content_as_stream(client, None, &input_source_category, category_file_path),
@@ -248,6 +374,9 @@ pub async fn get_xtream_playlist(cfg: &Arc<Config>, client: &reqwest::Client, in
Ok(sub_playlist_parsed) => {
if let Some(mut xtream_sub_playlist) = sub_playlist_parsed {
playlist_groups.append(&mut xtream_sub_playlist);
} else {
error!("Could not parse playlist {xtream_cluster} for input {}: {}",
input_source.name, sanitize_sensitive_info(&input_source.url));
}
}
Err(err) => errors.push(err)
@@ -271,7 +400,8 @@ async fn check_alias_user_state(cfg: &Arc<Config>, client: &reqwest::Client, inp
if let Some(aliases) = input.aliases.as_ref() {
for alias in aliases {
if is_input_expired(alias.exp_date) {
notify_account_expire(alias.exp_date, cfg, client, alias.username.as_ref().map_or("", |s| s.as_str()), &alias.name).await;
notify_account_expire(alias.exp_date, cfg, client, alias.username.as_ref()
.map_or("", |s| s.as_str()), &alias.name).await;
}
}
}
@@ -316,14 +446,18 @@ async fn check_alias_user_state(cfg: &Arc<Config>, client: &reqwest::Client, inp
// });
}
pub fn create_vod_info_from_item(target: &ConfigTarget, user: &ProxyUserCredentials, pli: &XtreamPlaylistItem, last_updated: i64) -> String {
pub fn create_vod_info_from_item(target: &ConfigTarget, user: &ProxyUserCredentials, pli: &XtreamPlaylistItem) -> String {
let category_id = pli.category_id;
let stream_id = if user.proxy.is_redirect(pli.item_type) || target.is_force_redirect(pli.item_type) { pli.provider_id } else { pli.virtual_id };
let added = pli.additional_properties.as_ref().and_then(StreamProperties::get_last_modified).unwrap_or(0);
let name = &pli.name;
let extension = pli.get_additional_property("container_extension")
.map_or_else(|| extract_extension_from_url(&pli.url).map_or_else(String::new, std::string::ToString::to_string),
|v| get_string_from_serde_value(&v).unwrap_or_default());
let added = last_updated / 1000;
let extension = pli
.get_container_extension()
.as_deref()
.filter(|ce| !ce.is_empty())
.or_else(|| extract_extension_from_url(&pli.url))
.map_or_else(String::new, ToString::to_string);
format!(r#"{{
"info": {{}},
"movie_data": {{
backend/src/utils/telegram.rs
+2 -2
View File
@@ -40,10 +40,10 @@ fn get_send_message_parse_mode_str(mode: &SendMessageParseMode) -> &'static str
#[derive(Debug, serde::Serialize)]
struct RequestObj {
pub chat_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub message_thread_id: Option<String>,
pub text: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub parse_mode: Option<String>,
}
config/config.yml
-10
View File
@@ -113,16 +113,6 @@ library:
formats:
# - "nfo" # Optionally write Kodi-compatible NFO files
# Classification rules for movies vs series
classification:
# Regex patterns to identify series/TV shows
# If a file/folder name matches any pattern, it's classified as a series
series_patterns:
- "(?i)s\\d+e\\d+" # S01E02 format
- "(?i)\\d+x\\d+" # 1x02 format
- "(?i)season[\\s._-]*\\d+" # "Season 1" format
- "(?i)episode[\\s._-]*\\d+" # "Episode 1" format
# Playlist integration
playlist:
# Group name for movies in M3U/Xtream playlist
frontend/Cargo.toml
+2 -2
View File
@@ -1,11 +1,11 @@
[package]
name = "frontend"
version = "3.2.24"
version = "3.2.25"
edition = "2021"
rust-version = "1.87.0"
[dependencies]
shared = { version = "3.2.24", path = "../shared" }
shared = { version = "3.2.25", path = "../shared" }
chrono = "0"
yew = "0.21"
yew-router = "0.18"
shared/Cargo.toml
+4 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "shared"
version = "3.2.24"
version = "3.2.25"
edition = "2021"
rust-version = "1.87.0"
@@ -23,6 +23,9 @@ zeroize = "1"
chrono = "0.4.42"
bytes = "1"
ciborium = "0.2.2"
hex = "0.4.3"
uuid = { version = "1.19.0", features = ["v4"] }
lz4_flex = "0.12.0"
[target.'cfg(target_arch = "wasm32")'.dependencies]
js-sys = "0.3.82"
shared/src/model/config/api_proxy.rs
+2 -2
View File
@@ -14,11 +14,11 @@ pub struct ApiProxyServerInfoDto {
pub name: String,
pub protocol: String,
pub host: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub port: Option<String>,
pub timezone: String,
pub message: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub path: Option<String>,
}
shared/src/model/config/api_user.rs
+7 -7
View File
@@ -14,25 +14,25 @@ pub enum UserConnectionPermission {
pub struct ProxyUserCredentialsDto {
pub username: String,
pub password: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub token: Option<String>,
#[serde(default = "ProxyType::default")]
pub proxy: ProxyType,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub server: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub epg_timeshift: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub created_at: Option<i64>,
#[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")]
#[serde(default, deserialize_with = "deserialize_timestamp")]
pub exp_date: Option<i64>,
#[serde(default)]
pub max_connections: u32,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub status: Option<ProxyUserStatus>,
#[serde(default = "default_as_true")]
pub ui_enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub comment: Option<String>,
}
shared/src/model/config/base.rs
+20 -20
View File
@@ -12,25 +12,25 @@ pub struct ConfigDto {
pub process_parallel: bool,
pub api: ConfigApiDto,
pub working_dir: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub backup_dir: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub user_config_dir: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub mapping_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub custom_stream_response_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub video: Option<VideoConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub schedules: Option<Vec<ScheduleConfigDto>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub log: Option<LogConfigDto>,
#[serde(default)]
pub user_access_control: bool,
#[serde(default = "default_connect_timeout_secs")]
pub connect_timeout_secs: u32,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub sleep_timer_mins: Option<u32>,
#[serde(default)]
pub update_on_boot: bool,
@@ -40,17 +40,17 @@ pub struct ConfigDto {
pub accept_insecure_ssl_certificates: bool,
#[serde(default)]
pub web_ui: Option<WebUiConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub messaging: Option<MessagingConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub reverse_proxy: Option<ReverseProxyConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub hdhomerun: Option<HdHomeRunConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub proxy: Option<ProxyConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub ipcheck: Option<IpCheckConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub library: Option<LibraryConfigDto>,
}
@@ -61,19 +61,19 @@ pub struct MainConfigDto {
#[serde(default)]
pub process_parallel: bool,
pub working_dir: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub backup_dir: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub user_config_dir: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub mapping_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub custom_stream_response_path: Option<String>,
#[serde(default)]
pub user_access_control: bool,
#[serde(default = "default_connect_timeout_secs")]
pub connect_timeout_secs: u32,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub sleep_timer_mins: Option<u32>,
#[serde(default)]
pub update_on_boot: bool,
@@ -125,7 +125,7 @@ impl From<&ConfigDto> for MainConfigDto {
// It has no other purpose than editing and saving the schedules
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)]
pub struct SchedulesConfigDto {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub schedules: Option<Vec<ScheduleConfigDto>>,
}
shared/src/model/config/cache.rs
+3 -3
View File
@@ -10,15 +10,15 @@ use path_clean::PathClean;
pub struct CacheConfigDto {
#[serde(default)]
pub enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub size: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub dir: Option<String>,
}
impl CacheConfigDto {
pub fn is_empty(&self) -> bool {
!self.enabled && is_blank_optional_string(&self.size) && is_blank_optional_string(&self.dir)
!self.enabled && is_blank_optional_string(self.size.as_deref()) && is_blank_optional_string(self.dir.as_deref())
}
pub(crate) fn prepare(&mut self, working_dir: &str) -> Result<(), TuliproxError> {
shared/src/model/config/epg.rs
+2 -2
View File
@@ -27,9 +27,9 @@ impl EpgSourceDto {
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Default)]
#[serde(deny_unknown_fields)]
pub struct EpgConfigDto {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub sources: Option<Vec<EpgSourceDto>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub smart_match: Option<EpgSmartMatchConfigDto>,
#[serde(skip)]
pub t_sources: Vec<EpgSourceDto>,
shared/src/model/config/epg_smart_match.rs
+1 -1
View File
@@ -33,7 +33,7 @@ pub struct EpgSmartMatchConfigDto {
#[serde(default)]
pub enabled: bool,
pub normalize_regex: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub strip: Option<Vec<String>>,
#[serde(default)]
pub name_prefix: EpgNamePrefix,
shared/src/model/config/hdhomerun.rs
+2 -7
View File
@@ -1,6 +1,6 @@
use crate::create_tuliprox_error_result;
use crate::error::{TuliproxError, TuliproxErrorKind};
use crate::utils::{default_as_true, generate_hdhr_device_id, generate_hdhr_device_id_from_base, hash_string, hex_encode, validate_hdhr_device_id};
use crate::utils::{default_as_true, generate_hdhr_device_id, generate_hdhr_device_id_from_base, hash_string, validate_hdhr_device_id};
use log::warn;
use std::collections::HashSet;
@@ -82,12 +82,7 @@ impl HdHomeRunDeviceConfigDto {
if self.device_udn == default_device_udn() || self.device_udn.is_empty() {
let hash = hash_string(&self.name);
// Format the hash into a valid UUID string
let p1 = hex_encode(&hash[0..4]);
let p2 = hex_encode(&hash[4..6]);
let p3 = hex_encode(&hash[6..8]);
let p4 = hex_encode(&hash[8..10]);
let p5 = hex_encode(&hash[10..16]);
self.device_udn = format!("{p1}-{p2}-{p3}-{p4}-{p5}");
self.device_udn = hash.to_valid_uuid();
if include_computed {
warn!("HDHomeRun device '{}' is missing a unique device_udn. A new one has been generated: {}", self.name, self.device_udn);
}
shared/src/model/config/input.rs
+16 -17
View File
@@ -200,9 +200,9 @@ impl ConfigInputOptionsDto {
pub struct StagedInputDto {
pub name: String,
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub username: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub password: Option<String>,
#[serde(default)]
pub method: InputFetchMethod,
@@ -240,15 +240,15 @@ pub struct ConfigInputAliasDto {
pub id: u16,
pub name: String,
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub username: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub password: Option<String>,
#[serde(default)]
pub priority: i16,
#[serde(default)]
pub max_connections: u16,
#[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")]
#[serde(default, deserialize_with = "deserialize_timestamp")]
pub exp_date: Option<i64>,
}
@@ -284,19 +284,19 @@ pub struct ConfigInputDto {
pub headers: HashMap<String, String>,
#[serde(default)]
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub epg: Option<EpgConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub username: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub password: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub persist: Option<String>,
#[serde(default = "default_as_true")]
pub enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub options: Option<ConfigInputOptionsDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub aliases: Option<Vec<ConfigInputAliasDto>>,
#[serde(default)]
pub priority: i16,
@@ -304,11 +304,11 @@ pub struct ConfigInputDto {
pub max_connections: u16,
#[serde(default)]
pub method: InputFetchMethod,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub staged: Option<StagedInputDto>,
#[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")]
#[serde(default, deserialize_with = "deserialize_timestamp")]
pub exp_date: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub panel_api: Option<PanelApiConfigDto>,
}
@@ -341,9 +341,8 @@ impl ConfigInputDto {
#[allow(clippy::cast_possible_truncation)]
pub fn prepare(&mut self, index: u16, _include_computed: bool) -> Result<u16, TuliproxError> {
let is_batch = matches!(self.input_type, InputType::M3uBatch | InputType::XtreamBatch);
self.name = self.name.trim().to_owned();
if self.name.is_empty() && !is_batch {
if self.name.is_empty() {
return Err(info_err!("name for input is mandatory".to_owned()));
}
@@ -356,7 +355,7 @@ impl ConfigInputDto {
}
}
self.persist = get_trimmed_string(&self.persist);
self.persist = get_trimmed_string(self.persist.as_deref());
let mut current_index = index + 1;
self.id = current_index;
shared/src/model/config/ipcheck.rs
+10 -10
View File
@@ -6,33 +6,33 @@ use regex::Regex;
#[serde(deny_unknown_fields)]
pub struct IpCheckConfigDto {
/// URL that may return both IPv4 and IPv6 in one response
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub url: Option<String>,
/// Dedicated URL to fetch only IPv4
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub url_ipv4: Option<String>,
/// Dedicated URL to fetch only IPv6
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub url_ipv6: Option<String>,
/// Optional regex pattern to extract IPv4
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub pattern_ipv4: Option<String>,
/// Optional regex pattern to extract IPv6
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub pattern_ipv6: Option<String>,
}
impl IpCheckConfigDto {
pub fn is_empty(&self) -> bool {
is_blank_optional_string(&self.url)
&& is_blank_optional_string(&self.url_ipv4)
&& is_blank_optional_string(&self.url_ipv6)
&& is_blank_optional_string(&self.pattern_ipv4)
&& is_blank_optional_string(&self.pattern_ipv6)
is_blank_optional_string(self.url.as_deref())
&& is_blank_optional_string(self.url_ipv4.as_deref())
&& is_blank_optional_string(self.url_ipv6.as_deref())
&& is_blank_optional_string(self.pattern_ipv4.as_deref())
&& is_blank_optional_string(self.pattern_ipv6.as_deref())
}
pub fn clean(&mut self) {
shared/src/model/config/library.rs
+1 -23
View File
@@ -5,7 +5,6 @@ use crate::utils::default_as_true;
pub fn default_metadata_path() -> String {
"library_metadata".to_string()
}
fn default_tmdb_rate_limit_ms() -> u64 {
250
}
@@ -24,15 +23,6 @@ fn default_movie_category() -> String {
fn default_series_category() -> String {
"Local TV Shows".to_string()
}
fn default_series_patterns() -> Vec<String> {
[
r"S\d{2}E\d{2}",
r"s\d{2}e\d{2}",
r"\d{1,2}x\d{1,2}",
r"Season\s*\d+",
r"Episode\s*\d+",
].iter().map(ToString::to_string).collect()
}
fn default_supported_extensions() -> Vec<String> {
[
"mp4",
@@ -57,8 +47,6 @@ pub struct LibraryConfigDto {
#[serde(default)]
pub metadata: LibraryMetadataConfigDto,
#[serde(default)]
pub classification: LibraryClassificationConfigDto,
#[serde(default)]
pub playlist: LibraryPlaylistConfigDto,
}
@@ -114,7 +102,7 @@ pub struct LibraryMetadataReadConfigDto {
pub struct LibraryTmdbConfigDto {
#[serde(default)]
pub enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub api_key: Option<String>,
#[serde(default = "default_tmdb_rate_limit_ms")]
pub rate_limit_ms: u64,
@@ -130,15 +118,6 @@ pub enum LibraryMetadataFormat {
Nfo,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(deny_unknown_fields)]
pub struct LibraryClassificationConfigDto {
#[serde(default = "default_series_patterns")]
pub series_patterns: Vec<String>,
#[serde(default)]
pub series_directory_patterns: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(deny_unknown_fields)]
pub struct LibraryPlaylistConfigDto {
@@ -148,7 +127,6 @@ pub struct LibraryPlaylistConfigDto {
pub series_category: String,
}
impl LibraryConfigDto {
pub fn prepare(&mut self) -> Result<(), TuliproxError> {
// Validate enabled state
shared/src/model/config/log.rs
+3 -3
View File
@@ -7,7 +7,7 @@ pub struct LogConfigDto {
pub sanitize_sensitive_info: bool,
#[serde(default)]
pub log_active_user: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub log_level: Option<String>,
}
@@ -23,11 +23,11 @@ impl Default for LogConfigDto {
impl LogConfigDto {
pub fn is_empty(&self) -> bool {
self.sanitize_sensitive_info && !self.log_active_user && is_blank_optional_string(&self.log_level)
self.sanitize_sensitive_info && !self.log_active_user && is_blank_optional_string(self.log_level.as_deref())
}
pub fn clean(&mut self) {
if is_blank_optional_string(&self.log_level) {
if is_blank_optional_string(self.log_level.as_deref()) {
self.log_level = None;
}
}
shared/src/model/config/macros.rs
+2 -2
View File
@@ -8,8 +8,8 @@ macro_rules! check_input_credentials {
return Err(info_err!("url for input is mandatory".to_string()));
}
$this.username = $crate::utils::get_trimmed_string(&$this.username);
$this.password = $crate::utils::get_trimmed_string(&$this.password);
$this.username = $crate::utils::get_trimmed_string($this.username.as_deref());
$this.password = $crate::utils::get_trimmed_string($this.password.as_deref());
}
match $input_type {
InputType::M3u => {
shared/src/model/config/messaging.rs
+4 -4
View File
@@ -38,7 +38,7 @@ pub struct PushoverMessagingConfigDto {
impl PushoverMessagingConfigDto {
pub fn is_empty(&self) -> bool {
is_blank_optional_string(&self.url)
is_blank_optional_string(self.url.as_deref())
&& self.token.trim().is_empty()
&& self.user.trim().is_empty()
}
@@ -49,11 +49,11 @@ impl PushoverMessagingConfigDto {
pub struct MessagingConfigDto {
#[serde(default)]
pub notify_on: Vec<MsgKind>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub telegram: Option<TelegramMessagingConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub rest: Option<RestMessagingConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub pushover: Option<PushoverMessagingConfigDto>,
}
shared/src/model/config/panel_api.rs
+1 -1
View File
@@ -24,7 +24,7 @@ pub struct PanelApiConfigDto {
#[serde(default = "default_as_true")]
pub enabled: bool,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub api_key: Option<String>,
#[serde(default)]
pub query_parameter: PanelApiQueryParametersDto,
shared/src/model/config/paths.rs
+1 -1
View File
@@ -5,6 +5,6 @@ pub struct ConfigPaths {
pub sources_file_path: String,
pub mapping_file_path: Option<String>,
pub api_proxy_file_path: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub custom_stream_response_path: Option<String>,
}
shared/src/model/config/proxy.rs
+4 -4
View File
@@ -11,16 +11,16 @@ pub struct ProxyConfigDto {
impl ProxyConfigDto {
pub fn is_empty(&self) -> bool {
is_blank_optional_string(&self.username)
&& is_blank_optional_string(&self.password)
is_blank_optional_string(self.username.as_deref())
&& is_blank_optional_string(self.password.as_deref())
&& self.url.trim().is_empty()
}
pub fn clean(&mut self) {
if is_blank_optional_string(&self.username) {
if is_blank_optional_string(self.username.as_deref()) {
self.username = None;
}
if is_blank_optional_string(&self.password) {
if is_blank_optional_string(self.password.as_deref()) {
self.password = None;
}
}
shared/src/model/config/reverse_proxy.rs
+6 -6
View File
@@ -36,17 +36,17 @@ pub struct ReverseProxyConfigDto {
#[serde(default)]
pub resource_rewrite_disabled: bool,
pub rewrite_secret: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub resource_retry: Option<ResourceRetryConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub disabled_header: Option<ReverseProxyDisabledHeaderConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub stream: Option<StreamConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub cache: Option<CacheConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub rate_limit: Option<RateLimitConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub geoip: Option<GeoIpConfigDto>,
}
shared/src/model/config/schedule.rs
+1 -1
View File
@@ -3,6 +3,6 @@
pub struct ScheduleConfigDto {
#[serde(default)]
pub schedule: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub targets: Option<Vec<String>>,
}
shared/src/model/config/sort.rs
+4 -4
View File
@@ -42,7 +42,7 @@ impl Display for SortOrder {
#[serde(deny_unknown_fields)]
pub struct ConfigSortGroupDto {
pub order: SortOrder,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub sequence: Option<Vec<String>>,
#[serde(skip)]
pub t_sequence: Option<Vec<Regex>>,
@@ -85,7 +85,7 @@ pub struct ConfigSortChannelDto {
// match against group title
pub group_pattern: String,
pub order: SortOrder,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub sequence: Option<Vec<String>>,
#[serde(skip)]
pub t_sequence: Option<Vec<Regex>>,
@@ -136,9 +136,9 @@ impl ConfigSortChannelDto {
pub struct ConfigSortDto {
#[serde(default)]
pub match_as_ascii: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub groups: Option<ConfigSortGroupDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub channels: Option<Vec<ConfigSortChannelDto>>,
}
shared/src/model/config/source.rs
+1 -1
View File
@@ -37,7 +37,7 @@ impl ConfigSourceDto {
#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct SourcesConfigDto {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub templates: Option<Vec<PatternTemplate>>,
pub sources: Vec<ConfigSourceDto>,
}
shared/src/model/config/stream.rs
+2 -2
View File
@@ -32,9 +32,9 @@ impl StreamBufferConfigDto {
pub struct StreamConfigDto {
#[serde(default)]
pub retry: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub buffer: Option<StreamBufferConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub throttle: Option<String>,
#[serde(default = "default_grace_period_millis")]
pub grace_period_millis: u64,
shared/src/model/config/target.rs
+15 -15
View File
@@ -13,7 +13,7 @@ pub struct ConfigTargetOptions {
pub share_live_streams: bool,
#[serde(default)]
pub remove_duplicates: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub force_redirect: Option<ClusterFlags>,
}
@@ -44,9 +44,9 @@ pub struct XtreamTargetOutputDto {
pub resolve_vod: bool,
#[serde(default = "default_resolve_delay_secs")]
pub resolve_vod_delay: u16,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub trakt: Option<TraktConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub filter: Option<String>,
#[serde(skip)]
pub t_filter: Option<Filter>,
@@ -94,13 +94,13 @@ impl XtreamTargetOutputDto {
#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct M3uTargetOutputDto {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub filename: Option<String>,
#[serde(default)]
pub include_type_in_url: bool,
#[serde(default)]
pub mask_redirect_url: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub filter: Option<String>,
#[serde(skip)]
pub t_filter: Option<Filter>,
@@ -127,7 +127,7 @@ impl M3uTargetOutputDto {
#[serde(deny_unknown_fields)]
pub struct StrmTargetOutputDto {
pub directory: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub username: Option<String>,
#[serde(default)]
pub style: StrmExportStyle,
@@ -137,9 +137,9 @@ pub struct StrmTargetOutputDto {
pub underscore_whitespace: bool,
#[serde(default)]
pub cleanup: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub strm_props: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub filter: Option<String>,
#[serde(skip)]
pub t_filter: Option<Filter>,
@@ -161,7 +161,7 @@ impl StrmTargetOutputDto {
pub struct HdHomeRunTargetOutputDto {
pub device: String,
pub username: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub use_output: Option<TargetType>,
}
@@ -204,22 +204,22 @@ pub struct ConfigTargetDto {
pub enabled: bool,
#[serde(default = "default_as_default")]
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub options: Option<ConfigTargetOptions>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub sort: Option<ConfigSortDto>,
pub filter: String,
#[serde(default)]
pub output: Vec<TargetOutputDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub rename: Option<Vec<ConfigRenameDto>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub mapping: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub favourites: Option<Vec<ConfigFavouritesDto>>,
#[serde(default)]
pub processing_order: ProcessingOrder,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub watch: Option<Vec<String>>,
#[serde(default)]
pub use_memory_cache: bool,
shared/src/model/config/video_download.rs
+7 -7
View File
@@ -12,11 +12,11 @@ pub const DEFAULT_VIDEO_EXTENSIONS: [&str; 6] = ["mkv", "avi", "mp4", "mpeg", "d
pub struct VideoDownloadConfigDto {
#[serde(default)]
pub headers: HashMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub directory: Option<String>,
#[serde(default)]
pub organize_into_directories: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub episode_pattern: Option<String>,
}
@@ -24,8 +24,8 @@ impl VideoDownloadConfigDto {
pub fn is_empty(&self) -> bool {
!self.organize_into_directories
&& self.headers.is_empty()
&& is_blank_optional_string(&self.directory)
&& is_blank_optional_string(&self.episode_pattern)
&& is_blank_optional_string(self.directory.as_deref())
&& is_blank_optional_string(self.episode_pattern.as_deref())
}
}
@@ -34,16 +34,16 @@ impl VideoDownloadConfigDto {
pub struct VideoConfigDto {
#[serde(default)]
pub extensions: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub download: Option<VideoDownloadConfigDto>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub web_search: Option<String>,
}
impl VideoConfigDto {
pub fn is_empty(&self) -> bool {
self.extensions.is_empty() && is_blank_optional_string(&self.web_search)
self.extensions.is_empty() && is_blank_optional_string(self.web_search.as_deref())
&& (self.download.is_none() || self.download.as_ref().is_some_and(|d| d.is_empty()))
}

Some files were not shown because too many files have changed in this diff Show More