From 3c4e18f996d5d625354e9fd89c7bece3b32d2eaf Mon Sep 17 00:00:00 2001 From: euzu <33094714+euzu@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:00:49 +0100 Subject: [PATCH] fast get_series, memory-cache lists, consistent sort order (#952) * fast get_series, memory-cache lists, consistent sort order - skip series episode details on list decode and in memory cache - serve Xtream lists from memory cache in canonical order - keep sorted index valid after resolve writes - cache verify_full per database identity - resource proxy: single attempt, 15s timeout, 6 fetches per host --- CHANGELOG.md | 20 + backend/app/src/api/api_utils/mod.rs | 11 + .../app/src/api/api_utils/resource_limiter.rs | 35 ++ backend/app/src/api/api_utils/resources.rs | 74 ++- backend/app/src/api/api_utils/streaming.rs | 8 - .../src/api/api_utils/tests/resource_proxy.rs | 242 +++++++- .../app/src/api/endpoints/hdhomerun_api.rs | 60 +- .../hls_api/tests/support/recording.rs | 2 +- .../app/src/api/endpoints/xtream_api/mod.rs | 32 +- .../app/src/api/endpoints/xtream_api/tests.rs | 35 +- backend/btree/src/lib.rs | 2 + backend/btree/src/sorted_index.rs | 336 ++++++++++- backend/btree/src/v3/mod.rs | 2 + backend/btree/src/v3/tree.rs | 29 +- backend/btree/src/v3/tree/bulk_store.rs | 4 +- backend/btree/src/v3/tree/error.rs | 2 +- backend/btree/src/v3/tree/memory.rs | 24 +- backend/btree/src/v3/tree/query.rs | 2 +- backend/btree/src/v3/tree/tests.rs | 1 + .../btree/src/v3/tree/tests/verified_cache.rs | 241 ++++++++ backend/btree/src/v3/tree/update.rs | 44 +- backend/btree/src/v3/tree/verified_cache.rs | 66 +++ backend/btree/src/v3/tree/verify.rs | 11 + backend/btree/src/v3/wal.rs | 38 +- backend/core/src/utils/network/request.rs | 1 + .../src/utils/network/request/download.rs | 5 + .../core/src/utils/network/request/options.rs | 6 +- backend/curation/src/kernel.rs | 5 +- backend/iptv/src/xtream/mod.rs | 46 +- backend/metadata/src/manager.rs | 2 +- backend/metadata/src/manager/publication.rs | 15 +- .../src/processor/playlist/target.rs | 4 +- backend/repository/src/lineup_guide_index.rs | 8 +- backend/repository/src/m3u_repository.rs | 9 +- backend/repository/src/playlist_mem_cache.rs | 294 +++++++++- .../src/playlist_repository/target.rs | 43 +- .../src/playlist_repository/tests/storage.rs | 547 ++++++++++++++++++ backend/repository/src/playlist_stream.rs | 52 +- backend/repository/src/test_config.rs | 15 + .../src/xtream_playlist_iterator.rs | 360 ++++++++++-- .../repository/src/xtream_repository/read.rs | 376 ++++++------ .../src/xtream_repository/target_publish.rs | 63 +- docs/src/configuration/reverse-proxy.md | 9 +- docs/src/configuration/source.md | 2 +- .../app/components/playlist/epg_view/mod.rs | 2 +- shared/src/foundation/filter/eval.rs | 5 +- shared/src/model/playlist.rs | 3 +- shared/src/model/playlist/kind.rs | 5 + shared/src/model/playlist/tests/identity.rs | 10 + shared/src/model/playlist/tests/xtream.rs | 264 +++++++++ shared/src/model/playlist/xtream.rs | 32 +- shared/src/model/stream_properties.rs | 7 + shared/src/model/stream_properties/series.rs | 73 ++- 53 files changed, 3047 insertions(+), 537 deletions(-) create mode 100644 backend/app/src/api/api_utils/resource_limiter.rs create mode 100644 backend/btree/src/v3/tree/tests/verified_cache.rs create mode 100644 backend/btree/src/v3/tree/verified_cache.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 850c637fc..48844f5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1073,6 +1073,15 @@ ## ⚙️ Optimizations +- **No full database re-verification on in-place writes.** In-place database writes (stream/series info resolves, probes) + no longer re-verify the whole database file when its identity was already verified in this process. + +- **`get_series` no longer decodes episode details for the list.** The series catalog list response only needs + the series header fields. Reading stored series items now skips deserializing and interning season and episode details + via a scoped decode projection, reducing response latency on large series catalogs. Xtream list endpoints are served + from the memory cache when `use_memory_cache` is enabled, and the memory cache no longer holds series season/episode + details (series info is read from disk). + - **Dependency updates.** `jsonwebtoken` 11, `quick-xml` 0.42, `tower-http` 0.7, `tokio-tungstenite` 0.30, `sysinfo` 0.39, `lol_html` 3, `lru` 0.19, `fancy-regex` 0.20 and `gloo-net` 0.7. The vendored `reqwest` moved to 0.13.5 with the request-extension forwarding patch re-applied. @@ -1247,6 +1256,17 @@ ## 🐛 Fixes +- **Image/resource proxy fails fast on upstream errors and limits per-host concurrency.** The image/resource proxy + no longer retries a failing upstream image three times (each failing image could block a browser slot for minutes), + fails after 15 s waiting for the response (the limit applies to waiting for the response, not to the image transfer), + and limits concurrent upstream fetches per host to 6 to avoid provider overload. + +- **Xtream list order stays sorted after stream or series info resolves.** Resolving VOD or series info updates the + target database in place, and every commit removed the sorted index, so later list requests returned items in + virtual-id order. The Xtream target write path now keeps the index (new or moved keys are merged in) and reads values + by primary key while the index is older than the database. Disk and memory cache list the same items in the same + `(source_ordinal, virtual_id)` order. + - **Filters are no longer limited by the parser stack.** A filter that expanded to more than a few dozen terms was rejected with `stack limit reached` on the static-musl build and left the container restarting. The parser dependency no longer probes the stack on musl, and a filter is now stored flat and read, rendered, evaluated and dropped without diff --git a/backend/app/src/api/api_utils/mod.rs b/backend/app/src/api/api_utils/mod.rs index 5eea8f86c..25d023612 100644 --- a/backend/app/src/api/api_utils/mod.rs +++ b/backend/app/src/api/api_utils/mod.rs @@ -212,6 +212,8 @@ where /// requests a provider or EPG entry can trigger through one resource link stays bounded. const RESOURCE_REDIRECT_LIMIT: u8 = 5; +pub(crate) const RESOURCE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15); + /// How the client-visible answer of a resource route is shaped. /// /// The variant mirrors the classification of the destination the route serves: a [`Public`] destination @@ -241,6 +243,7 @@ mod media_server; mod metering; mod provider_acquire; mod provider_urls; +mod resource_limiter; mod resources; mod response; mod serialization; @@ -340,3 +343,11 @@ pub(crate) use self::{ resolve_stream_user_agent_index, should_allow_exhausted_shared_reconnect, stream_response, }, }; +#[cfg(test)] +use self::{ + resource_limiter::{ + acquire_resource_host_permit, get_resource_host_semaphore, RESOURCE_HOST_CONCURRENCY_LIMIT, + RESOURCE_HOST_MAP_PRUNE_THRESHOLD, + }, + resources::fetch_resource_with_timeout, +}; diff --git a/backend/app/src/api/api_utils/resource_limiter.rs b/backend/app/src/api/api_utils/resource_limiter.rs new file mode 100644 index 000000000..0c98c1835 --- /dev/null +++ b/backend/app/src/api/api_utils/resource_limiter.rs @@ -0,0 +1,35 @@ +use std::{ + collections::HashMap, + sync::{Arc, LazyLock, Mutex}, +}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +pub(super) const RESOURCE_HOST_CONCURRENCY_LIMIT: usize = 6; +pub(super) const RESOURCE_HOST_MAP_PRUNE_THRESHOLD: usize = 256; + +static RESOURCE_HOST_LIMITERS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +pub(super) async fn acquire_resource_host_permit(host: &str) -> Option { + let semaphore = { + let Ok(mut guard) = RESOURCE_HOST_LIMITERS.lock() else { + return None; + }; + if let Some(semaphore) = guard.get(host) { + semaphore.clone() + } else { + if guard.len() > RESOURCE_HOST_MAP_PRUNE_THRESHOLD { + guard.retain(|_, semaphore| Arc::strong_count(semaphore) > 1); + } + let semaphore = Arc::new(Semaphore::new(RESOURCE_HOST_CONCURRENCY_LIMIT)); + guard.insert(host.to_string(), Arc::clone(&semaphore)); + semaphore + } + }; + semaphore.acquire_owned().await.ok() +} + +#[cfg(test)] +pub(super) fn get_resource_host_semaphore(host: &str) -> Option> { + RESOURCE_HOST_LIMITERS.lock().ok()?.get(host).cloned() +} diff --git a/backend/app/src/api/api_utils/resources.rs b/backend/app/src/api/api_utils/resources.rs index f2aedab49..245729b23 100644 --- a/backend/app/src/api/api_utils/resources.rs +++ b/backend/app/src/api/api_utils/resources.rs @@ -1,16 +1,19 @@ use super::{ force_stream_response, get_headers_from_request, get_hls_playback_ttl_secs, is_hop_by_hop_response_header, - media_server_image_error_status, open_media_server_image_resource, redirect, serve_file, try_unwrap_body, - ForceStreamRequestContext, HeaderFilter, ResourceFetchPolicy, StreamResponseMode, RESOURCE_REDIRECT_LIMIT, + media_server_image_error_status, open_media_server_image_resource, redirect, + resource_limiter::acquire_resource_host_permit, serve_file, try_unwrap_body, ForceStreamRequestContext, + HeaderFilter, ResourceFetchPolicy, StreamResponseMode, RESOURCE_ATTEMPT_TIMEOUT, RESOURCE_REDIRECT_LIMIT, }; use crate::{ api::model::{tee_stream, AppState, StreamDetails, StreamError, UserSession}, auth::Fingerprint, model::{ConfigInput, PlaybackKind}, utils::{ - async_file_writer, classify_output_resource_url, classify_resource_hop, create_new_file_for_write, - debug_if_enabled, request, - request::{classify_resource_destination, send_with_retry_and_provider, ResourceDestination}, + async_file_writer, classify_output_resource_url, classify_resource_hop, create_new_file_for_write, request, + request::{ + classify_resource_destination, send_with_retry_and_provider_policy_with_options, RequestFetchOptions, + ResourceDestination, + }, trace_if_enabled, LRUResourceCache, ResourceHop, }, }; @@ -25,7 +28,7 @@ use shared::{ model::{InputFetchMethod, PlaylistItemType, ResourceOutputPolicy, StreamChannel}, utils::sanitize_sensitive_info, }; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::sync::RwLock; use url::Url; @@ -185,6 +188,28 @@ pub(super) async fn fetch_resource_with_retry( resource_url: &str, req_headers: &HashMap>, input: Option<&ConfigInput>, +) -> Option { + fetch_resource_with_timeout( + app_state, + url, + fetch_policy, + resource_url, + req_headers, + input, + RESOURCE_ATTEMPT_TIMEOUT, + ) + .await +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn fetch_resource_with_timeout( + app_state: &Arc, + url: &Url, + fetch_policy: ResourceFetchPolicy, + resource_url: &str, + req_headers: &HashMap>, + input: Option<&ConfigInput>, + attempt_timeout: Duration, ) -> Option { let cache_key = fetch_policy.cache_key(resource_url); let config = app_state.app_config.config.load(); @@ -196,6 +221,9 @@ pub(super) async fn fetch_resource_with_retry( let mut current_headers = req_headers.clone(); let mut current_input = input; let mut method = input.map_or(InputFetchMethod::GET, |i| i.method); + let fetch_options = RequestFetchOptions::with_attempt_idle_timeout(attempt_timeout) + .without_resource_retries() + .with_header_timeout_only(); for redirects in 0..=RESOURCE_REDIRECT_LIMIT { // Every hop is routed on its own, so a redirect cannot move a request to an egress the hop @@ -207,13 +235,17 @@ pub(super) async fn fetch_resource_with_retry( } let use_proxy_aware_client = hop == ResourceHop::ViaProxy; let provider_config = current_input.and_then(|i| i.get_resolve_provider(current_url.as_str())); - let response = match send_with_retry_and_provider( + let permit = + if let Some(host) = current_url.host_str() { acquire_resource_host_permit(host).await } else { None }; + let result = send_with_retry_and_provider_policy_with_options( &app_state.app_config, ¤t_url, provider_config.as_ref(), // Hand redirects back instead of retrying them: the loop below follows them itself, so that // every hop is classified and no client-side redirect policy decides where the request ends. true, + true, + fetch_options, |resolved_url| { let http_client = if use_proxy_aware_client { app_state.http_clients.resource_public_no_redirect.load() @@ -231,8 +263,19 @@ pub(super) async fn fetch_resource_with_retry( ) }, ) - .await - { + .await; + drop(permit); + let response = match result { + // The single-attempt path hands HTTP error statuses back as responses; treat them as a failed + // fetch, as the retrying path did. + Ok(response) if response.status().is_client_error() || response.status().is_server_error() => { + debug!( + "Resource fetch failed for {}: status {}", + sanitize_sensitive_info(resource_url), + response.status() + ); + return None; + } Ok(response) => response, Err(err) => { debug!( @@ -279,18 +322,7 @@ pub(super) async fn fetch_resource_with_retry( ); } - debug_if_enabled!("Failed to open resource got status {status} for {}", sanitize_sensitive_info(resource_url)); - if !fetch_policy.relays_upstream_response() { - return Some(status.into_response()); - } - let mut response_builder = axum::response::Response::builder().status(status); - for (key, value) in response.headers() { - if !is_hop_by_hop_response_header(key) { - response_builder = response_builder.header(key, value); - } - } - let stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)); - return Some(try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(stream)))); + return None; } None } diff --git a/backend/app/src/api/api_utils/streaming.rs b/backend/app/src/api/api_utils/streaming.rs index cea64b0d5..29f8b82b8 100644 --- a/backend/app/src/api/api_utils/streaming.rs +++ b/backend/app/src/api/api_utils/streaming.rs @@ -1186,14 +1186,6 @@ impl ResourceFetchPolicy { Self::NonPublic => None, } } - - /// Whether an upstream response may be relayed to the client. - /// - /// A public destination was chosen by the user's own configuration, so what it answered is what - /// the client asked for. A resource proxy whose destination is deliberately hidden from the client - /// must not become a reader for that destination: status, headers and body of an upstream error can - /// disclose more about the internal service than the link itself. - pub(super) const fn relays_upstream_response(self) -> bool { matches!(self, Self::Public) } } pub fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> bool { diff --git a/backend/app/src/api/api_utils/tests/resource_proxy.rs b/backend/app/src/api/api_utils/tests/resource_proxy.rs index 9a3414d60..122320e64 100644 --- a/backend/app/src/api/api_utils/tests/resource_proxy.rs +++ b/backend/app/src/api/api_utils/tests/resource_proxy.rs @@ -1,6 +1,8 @@ use super::{ - create_test_app_state, media_server_image_error_status, resource_input_for_url, resource_proxy_response, - resource_redirect_or_proxy, resource_response, spawn_legacy_hls_test_origin, ResourceFetchPolicy, + acquire_resource_host_permit, create_test_app_state, fetch_resource_with_timeout, get_resource_host_semaphore, + media_server_image_error_status, resource_input_for_url, resource_proxy_response, resource_redirect_or_proxy, + resource_response, spawn_legacy_hls_test_origin, ResourceFetchPolicy, RESOURCE_HOST_CONCURRENCY_LIMIT, + RESOURCE_HOST_MAP_PRUNE_THRESHOLD, }; use crate::{ media_server::{MediaServerError, MediaServerErrorKind}, @@ -13,12 +15,21 @@ use axum::{ }; use bytes::Bytes; use http_body_util::BodyExt; -use std::{fmt::Write as _, sync::Arc}; +use std::{ + collections::HashMap, + fmt::Write as _, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, sync::RwLock, }; +use url::Url; #[test] fn media_server_image_error_status_classifies_client_and_upstream_failures() { @@ -441,3 +452,228 @@ async fn resource_client_refuses_names_that_resolve_to_local_addresses() { } assert!(causes.contains("local to this host"), "{causes}"); } + +#[tokio::test] +async fn upstream_502_fails_fast_without_retrying() { + let app_state = create_test_app_state(); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock origin binds"); + let origin_addr = listener.local_addr().expect("mock origin address"); + let request_count = Arc::new(AtomicUsize::new(0)); + let request_count_task = Arc::clone(&request_count); + + let origin_task = tokio::spawn(async move { + loop { + let Ok(accepted) = tokio::time::timeout(Duration::from_millis(500), listener.accept()).await else { + break; + }; + let Ok((mut socket, _)) = accepted else { break }; + request_count_task.fetch_add(1, Ordering::SeqCst); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let Ok(read) = socket.read(&mut chunk).await else { break }; + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let head = "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + } + }); + + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{origin_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); + + let response = + resource_response(&app_state, ResourceFetchPolicy::Public, "http://8.8.8.8/icon.png", &HeaderMap::new(), None) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + let _ = origin_task.await; + assert_eq!(request_count.load(Ordering::SeqCst), 1, "failing upstream resource request must not be retried"); +} + +#[tokio::test] +async fn unresponsive_upstream_fails_after_attempt_timeout() { + let app_state = create_test_app_state(); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock origin binds"); + let origin_addr = listener.local_addr().expect("mock origin address"); + + let origin_task = tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + let mut chunk = [0_u8; 1024]; + let _ = socket.read(&mut chunk).await; + tokio::time::sleep(Duration::from_secs(60)).await; + } + }); + + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{origin_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); + + let url = Url::parse("http://8.8.8.8/unresponsive.png").expect("valid url"); + let start = std::time::Instant::now(); + let resp = fetch_resource_with_timeout( + &app_state, + &url, + ResourceFetchPolicy::Public, + "http://8.8.8.8/unresponsive.png", + &HashMap::new(), + None, + Duration::from_millis(100), + ) + .await; + + assert!(resp.is_none(), "fetch must fail after attempt timeout"); + assert!(start.elapsed() < Duration::from_secs(5), "fetch should time out quickly based on attempt timeout"); + origin_task.abort(); +} + +#[tokio::test] +async fn resource_fetch_timeout_covers_headers_only_and_delivers_delayed_body() { + let app_state = create_test_app_state(); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock origin binds"); + let origin_addr = listener.local_addr().expect("mock origin address"); + + let origin_task = tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + let mut chunk = [0_u8; 1024]; + let _ = socket.read(&mut chunk).await; + let response_headers = + b"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 10\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(response_headers).await; + let _ = socket.flush().await; + // Delay body longer than the attempt timeout (100ms attempt timeout, delay 250ms) + tokio::time::sleep(Duration::from_millis(250)).await; + let _ = socket.write_all(b"0123456789").await; + let _ = socket.flush().await; + } + }); + + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{origin_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); + + let url = Url::parse("http://8.8.8.8/delayed_body.png").expect("valid url"); + let resp = fetch_resource_with_timeout( + &app_state, + &url, + ResourceFetchPolicy::Public, + "http://8.8.8.8/delayed_body.png", + &HashMap::new(), + None, + Duration::from_millis(100), + ) + .await; + + assert!(resp.is_some(), "fetch must succeed when headers arrive immediately"); + let response = resp.expect("valid response"); + assert_eq!(response.status(), StatusCode::OK); + + let body_bytes = response.into_body().collect().await.expect("reading response body must succeed").to_bytes(); + assert_eq!(&body_bytes[..], b"0123456789", "full body must be delivered despite body delay"); + origin_task.abort(); +} + +#[tokio::test] +async fn resource_limiter_blocks_excess_permits_on_same_host_without_blocking_other_hosts() { + use futures::FutureExt as _; + use std::pin::pin; + + let host_a = "limiter-host-a.invalid"; + let host_b = "limiter-host-b.invalid"; + + let mut permits_a = Vec::new(); + for _ in 0..RESOURCE_HOST_CONCURRENCY_LIMIT { + let permit = acquire_resource_host_permit(host_a).await; + assert!(permit.is_some(), "must acquire permit up to capacity"); + permits_a.push(permit.expect("permit")); + } + + // Another host is not blocked + let permit_b = acquire_resource_host_permit(host_b).await; + assert!(permit_b.is_some(), "another host must not be blocked when host A is at capacity"); + + // With all permits taken for host A, the next acquire waits + let mut next_acquire = pin!(acquire_resource_host_permit(host_a)); + assert!(next_acquire.as_mut().now_or_never().is_none(), "next acquire on host A must wait"); + + // Release one permit + drop(permits_a.pop()); + + // Now it completes + let acquired = tokio::time::timeout(Duration::from_secs(1), next_acquire) + .await + .expect("must resolve after permit is released"); + assert!(acquired.is_some(), "must acquire permit once one is released"); +} + +#[tokio::test] +async fn resource_limiter_prunes_idle_entries_above_threshold_while_keeping_active_host() { + use futures::FutureExt as _; + use std::pin::pin; + + let active_host = "limiter-prune-active.invalid"; + let held_permit = acquire_resource_host_permit(active_host).await; + assert!(held_permit.is_some(), "must acquire permit on active host"); + let active_semaphore = get_resource_host_semaphore(active_host).expect("active host semaphore must exist"); + + for i in 0..=RESOURCE_HOST_MAP_PRUNE_THRESHOLD { + let idle_host = format!("limiter-prune-idle-{i}.invalid"); + let permit = acquire_resource_host_permit(&idle_host).await; + assert!(permit.is_some(), "must acquire permit on idle host"); + drop(permit); + } + + let new_host = "limiter-prune-new.invalid"; + let new_permit = acquire_resource_host_permit(new_host).await; + assert!(new_permit.is_some(), "must acquire permit on new host"); + + // Idle host was pruned + assert!( + get_resource_host_semaphore("limiter-prune-idle-0.invalid").is_none(), + "idle entries above threshold must be pruned" + ); + + // Active host kept its semaphore (same Arc and permits still counted) + let current_active_semaphore = + get_resource_host_semaphore(active_host).expect("active host semaphore must still be present"); + assert!(Arc::ptr_eq(&active_semaphore, ¤t_active_semaphore), "active host must keep the same semaphore Arc"); + assert_eq!( + current_active_semaphore.available_permits(), + RESOURCE_HOST_CONCURRENCY_LIMIT - 1, + "active host permit count must still reflect held permit" + ); + + // Remaining capacity on active host can be acquired + let mut more_permits = Vec::new(); + for _ in 0..(RESOURCE_HOST_CONCURRENCY_LIMIT - 1) { + let permit = acquire_resource_host_permit(active_host).await; + assert!(permit.is_some(), "must acquire remaining capacity on active host"); + more_permits.push(permit.expect("permit")); + } + + // Now active host is saturated and waits + let mut next_acquire = pin!(acquire_resource_host_permit(active_host)); + assert!(next_acquire.as_mut().now_or_never().is_none(), "next acquire on active host must wait"); + + drop(held_permit); + let acquired = tokio::time::timeout(Duration::from_secs(1), next_acquire) + .await + .expect("must resolve after permit is released"); + assert!(acquired.is_some(), "must acquire permit once one is released"); + drop(new_permit); + drop(more_permits); +} diff --git a/backend/app/src/api/endpoints/hdhomerun_api.rs b/backend/app/src/api/endpoints/hdhomerun_api.rs index 932fab8e0..4c45f2a26 100644 --- a/backend/app/src/api/endpoints/hdhomerun_api.rs +++ b/backend/app/src/api/endpoints/hdhomerun_api.rs @@ -322,24 +322,24 @@ async fn lineup_status( 0 } } else if target.has_output(TargetType::Xtream) { - let cred = Arc::clone(&credentials); - let live = if cred.allows_cluster(XtreamCluster::Live) { - match XtreamPlaylistIterator::new(XtreamCluster::Live, &cfg, &target, None, &cred).await { - Ok(stream) => stream.filter_map(|entry| async move { entry.ok() }).count().await, - Err(_) => 0, + let mut count = 0; + for cluster in [XtreamCluster::Live, XtreamCluster::Video] { + if credentials.allows_cluster(cluster) { + if let Ok(stream) = XtreamPlaylistIterator::new( + cluster, + &cfg, + &app_state.app_state.playlists, + &target, + None, + &credentials, + ) + .await + { + count += stream.filter_map(|entry| async move { entry.ok() }).count().await; + } } - } else { - 0 - }; - let vod = if cred.allows_cluster(XtreamCluster::Video) { - match XtreamPlaylistIterator::new(XtreamCluster::Video, &cfg, &target, None, &cred).await { - Ok(stream) => stream.filter_map(|entry| async move { entry.ok() }).count().await, - Err(_) => 0, - } - } else { - 0 - }; - live + vod + } + count } else { 0 } @@ -416,12 +416,30 @@ async fn lineup( let (base_url_live, base_url_vod) = xtream_base_urls(credentials, target, server_info.get_base_url()); let live_channels = if credentials.allows_cluster(XtreamCluster::Live) { - XtreamPlaylistIterator::new(XtreamCluster::Live, cfg, target, None, credentials).await.ok() + XtreamPlaylistIterator::new( + XtreamCluster::Live, + cfg, + &app_state.app_state.playlists, + target, + None, + credentials, + ) + .await + .ok() } else { None }; let vod_channels = if credentials.allows_cluster(XtreamCluster::Video) { - XtreamPlaylistIterator::new(XtreamCluster::Video, cfg, target, None, credentials).await.ok() + XtreamPlaylistIterator::new( + XtreamCluster::Video, + cfg, + &app_state.app_state.playlists, + target, + None, + credentials, + ) + .await + .ok() } else { None }; @@ -783,7 +801,9 @@ async fn resolve_channel_stream_url( if !credentials.allows_cluster(cluster) { continue; } - let Ok(channels) = XtreamPlaylistIterator::new(cluster, cfg, target, None, credentials).await else { + let Ok(channels) = + XtreamPlaylistIterator::new(cluster, cfg, &app_state.playlists, target, None, credentials).await + else { continue; }; let Some((rank, item)) = find_channel(channels, query).await else { continue }; diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs b/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs index 3d771d2e5..cba72522d 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs @@ -142,7 +142,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn cache_recording_test_item .playlists .cache_playlist( &target.name, - PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { live, vod, series })), + PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage::from_trees(live, vod, series))), ) .await; } else { diff --git a/backend/app/src/api/endpoints/xtream_api/mod.rs b/backend/app/src/api/endpoints/xtream_api/mod.rs index ae8b42b5f..c59b486ea 100644 --- a/backend/app/src/api/endpoints/xtream_api/mod.rs +++ b/backend/app/src/api/endpoints/xtream_api/mod.rs @@ -1876,17 +1876,39 @@ async fn xtream_player_api( let result = match action { crate::model::XC_ACTION_GET_LIVE_STREAMS => skip_flag_optional!( skip_live, - xtream_load_rewrite_playlist(XtreamCluster::Live, &app_state.app_config, &target, category_id, &user).await + xtream_load_rewrite_playlist( + XtreamCluster::Live, + &app_state.app_config, + &app_state.playlists, + &target, + category_id, + &user + ) + .await ), crate::model::XC_ACTION_GET_VOD_STREAMS => skip_flag_optional!( skip_vod, - xtream_load_rewrite_playlist(XtreamCluster::Video, &app_state.app_config, &target, category_id, &user) - .await + xtream_load_rewrite_playlist( + XtreamCluster::Video, + &app_state.app_config, + &app_state.playlists, + &target, + category_id, + &user + ) + .await ), crate::model::XC_ACTION_GET_SERIES => skip_flag_optional!( skip_series, - xtream_load_rewrite_playlist(XtreamCluster::Series, &app_state.app_config, &target, category_id, &user) - .await + xtream_load_rewrite_playlist( + XtreamCluster::Series, + &app_state.app_config, + &app_state.playlists, + &target, + category_id, + &user + ) + .await ), _ => Some(Err(TuliproxError::ApiXtream(format!("Unknown api call: {action} for target: {}", target.name)))), }; diff --git a/backend/app/src/api/endpoints/xtream_api/tests.rs b/backend/app/src/api/endpoints/xtream_api/tests.rs index 2c8ddd5c8..7293a403a 100644 --- a/backend/app/src/api/endpoints/xtream_api/tests.rs +++ b/backend/app/src/api/endpoints/xtream_api/tests.rs @@ -201,32 +201,7 @@ async fn xtream_short_epg_returns_imported_ics_programme_for_matching_channel_id .expect("write target EPG"); let app_state = create_test_app_state(config); - let live_item = short_epg_live_item(); - let mut live = BPlusTree::new(); - live.insert(live_item.virtual_id.get(), live_item.clone()); - app_state - .playlists - .cache_playlist( - &target.name, - PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { - live, - vod: BPlusTree::new(), - series: BPlusTree::new(), - })), - ) - .await; - let mut id_mapping = BPlusTree::new(); - id_mapping.insert( - live_item.virtual_id, - VirtualIdRecord::new( - live_item.provider_id, - live_item.virtual_id, - PlaylistItemType::Live, - VirtualId::new(0), - UUIDType::default(), - ), - ); - app_state.playlists.cache_id_mapping(&target.name, id_mapping).await; + cache_xtream_test_item(&app_state, &target, short_epg_live_item(), None).await; let mut user = ProxyUserCredentials::default(); user.output_clusters = ClusterFlags::all(); @@ -303,11 +278,11 @@ async fn cache_xtream_test_item( .playlists .cache_playlist( &target.name, - PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { + PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage::from_trees( live, - vod: BPlusTree::new(), - series: BPlusTree::new(), - })), + BPlusTree::new(), + BPlusTree::new(), + ))), ) .await; let mut id_mapping = BPlusTree::new(); diff --git a/backend/btree/src/lib.rs b/backend/btree/src/lib.rs index 86ea6c748..f845e99cd 100644 --- a/backend/btree/src/lib.rs +++ b/backend/btree/src/lib.rs @@ -44,6 +44,8 @@ pub use recovery::{ RecoveryRepositoryState, RecoverySchema, RecoveryStoragePlacement, RecoveryVerificationReport, }; // --- Sorted-index sidecar -------------------------------------------------- +/// Snapshot of a sorted-index sidecar that the database writer rewrites after its own commit. +pub use sorted_index::v4::IndexSnapshot; /// Iterates a database in sorted-index order, falling back to the caller's own /// handling when the sidecar is unusable. pub use sorted_index::v4::OwnedIterator as SortedIndexIterator; diff --git a/backend/btree/src/sorted_index.rs b/backend/btree/src/sorted_index.rs index 4c5dc280e..5da76517c 100644 --- a/backend/btree/src/sorted_index.rs +++ b/backend/btree/src/sorted_index.rs @@ -1,7 +1,7 @@ pub mod v4 { use crate::{ codec::{binary_deserialize, binary_serialize}, - v3::{BPlusTreeQuery, Locator}, + v3::{publish_database, sync_parent_directory, temporary_path, BPlusTreeQuery, Locator}, }; use serde::{Deserialize, Serialize}; use std::{ @@ -28,7 +28,7 @@ pub mod v4 { fn invalid_integer(error: TryFromIntError) -> io::Error { invalid_data(error.to_string()) } - fn header(database_id: [u8; 16], generation: u64, count: u64) -> [u8; HEADER_LEN] { + pub(crate) fn header(database_id: [u8; 16], generation: u64, count: u64) -> [u8; HEADER_LEN] { let mut bytes = [0; HEADER_LEN]; bytes[0..4].copy_from_slice(MAGIC); bytes[4..8].copy_from_slice(&VERSION.to_le_bytes()); @@ -58,11 +58,14 @@ pub mod v4 { .ok_or_else(|| invalid_data("sorted-index u64 field is truncated")) } - fn decode_header( - mut bytes: [u8; HEADER_LEN], - expected_database_id: [u8; 16], - expected_generation: u64, - ) -> io::Result { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct HeaderIdentity { + database_id: [u8; 16], + generation: u64, + count: u64, + } + + fn decode_header_identity(mut bytes: [u8; HEADER_LEN]) -> io::Result { if bytes[0..4] != *MAGIC || read_u32(&bytes, 4)? != VERSION || read_u32(&bytes, 8)? != TREE_VERSION @@ -78,10 +81,27 @@ pub mod v4 { if crc32fast::hash(&bytes) != expected_checksum { return Err(invalid_data("sorted-index header checksum mismatch")); } - if bytes[16..32] != expected_database_id || read_u64(&bytes, 32)? != expected_generation { - return Err(invalid_data("sorted-index tree identity or generation mismatch")); + let mut database_id = [0u8; 16]; + database_id.copy_from_slice(&bytes[16..32]); + let generation = read_u64(&bytes, 32)?; + let count = read_u64(&bytes, 40)?; + Ok(HeaderIdentity { database_id, generation, count }) + } + + fn decode_header( + bytes: [u8; HEADER_LEN], + expected_database_id: [u8; 16], + expected_generation: u64, + ) -> io::Result<(u64, bool)> { + let identity = decode_header_identity(bytes)?; + if identity.database_id != expected_database_id { + return Err(invalid_data("sorted-index tree identity mismatch")); } - read_u64(&bytes, 40) + if identity.generation > expected_generation { + return Err(invalid_data("sorted-index generation is newer than tree generation")); + } + let is_stale = identity.generation < expected_generation; + Ok((identity.count, is_stale)) } pub struct Writer { @@ -153,6 +173,7 @@ pub mod v4 { remaining_entries: u64, remaining_bytes: u64, failed: bool, + stale: bool, body: Vec, _marker: PhantomData<(SortKey, K)>, } @@ -171,17 +192,26 @@ pub mod v4 { let mut input = BufReader::new(file); let mut encoded = [0; HEADER_LEN]; input.read_exact(&mut encoded)?; - let remaining_entries = decode_header(encoded, expected_database_id, expected_generation)?; - Ok(Self { + let (remaining_entries, stale) = decode_header(encoded, expected_database_id, expected_generation)?; + let remaining_bytes = length - HEADER_LEN as u64; + Ok(Self::from_reader(input, remaining_bytes, remaining_entries, stale)) + } + + fn from_reader(input: BufReader, remaining_bytes: u64, remaining_entries: u64, stale: bool) -> Self { + Self { input, remaining_entries, - remaining_bytes: length - HEADER_LEN as u64, + remaining_bytes, failed: false, + stale, body: Vec::new(), _marker: PhantomData, - }) + } } + #[inline] + pub(crate) fn is_stale(&self) -> bool { self.stale } + #[cfg(test)] pub(crate) fn remaining(&self) -> u64 { self.remaining_entries } @@ -278,6 +308,7 @@ pub mod v4 { reader: Reader, query: BPlusTreeQuery, previous_sort_key: Option, + is_stale: bool, finished: bool, } @@ -290,7 +321,8 @@ pub mod v4 { pub fn open(query: BPlusTreeQuery, index_path: &Path) -> io::Result { let (database_id, generation) = query.snapshot_identity(); let reader = Reader::open(index_path, database_id, generation)?; - Ok(Self { reader, query, previous_sort_key: None, finished: false }) + let is_stale = reader.is_stale(); + Ok(Self { reader, query, previous_sort_key: None, is_stale, finished: false }) } #[cfg(test)] @@ -309,30 +341,140 @@ pub mod v4 { if self.finished { return None; } - let result = match self.reader.read_next_borrowed() { - Ok(Some(entry)) => { - if self.previous_sort_key.as_ref().is_some_and(|previous| previous > &entry.sort_key) { - Err(invalid_data("sorted-index entries are out of order")) - } else { - let BorrowedEntry { sort_key, primary_key, serialized_primary_key, locator } = entry; - self.previous_sort_key = Some(sort_key); - self.query.read_locator_value(locator, serialized_primary_key).map(|value| (primary_key, value)) + loop { + let entry = match self.reader.read_next_borrowed() { + Ok(Some(entry)) => entry, + Ok(None) => { + self.finished = true; + return None; } - } - Ok(None) => { + Err(error) => { + self.finished = true; + return Some(Err(error)); + } + }; + + if self.previous_sort_key.as_ref().is_some_and(|previous| previous > &entry.sort_key) { self.finished = true; - return None; + return Some(Err(invalid_data("sorted-index entries are out of order"))); } - Err(error) => Err(error), - }; - match result { - Ok(entry) => Some(Ok(entry)), - Err(error) => Some(Err(error)), + + let BorrowedEntry { sort_key, primary_key, serialized_primary_key, locator } = entry; + self.previous_sort_key = Some(sort_key); + + if self.is_stale { + match self.query.query_io(&primary_key) { + Ok(Some(value)) => return Some(Ok((primary_key, value))), + Ok(None) => {} + Err(error) => { + self.finished = true; + return Some(Err(error)); + } + } + continue; + } + + let result = + self.query.read_locator_value(locator, serialized_primary_key).map(|value| (primary_key, value)); + return Some(result); } } } + + /// In-memory snapshot of a sorted index for read-modify-write workflows. + /// + /// Every `BPlusTreeUpdate` commit deletes the sorted index file. Therefore, a stale + /// index only exists when the caller holding the database write lock rewrote it via + /// `write_merged` right after its own commit, and it must contain every key of the tree. + #[derive(Debug)] + pub struct IndexSnapshot { + database_id: [u8; 16], + generation: u64, + entries: Vec<(SortKey, K, Locator)>, + } + + impl IndexSnapshot + where + SortKey: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + { + pub fn read(index_path: &Path) -> io::Result> { + let file = match File::open(index_path) { + Ok(file) => file, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err), + }; + let length = file.metadata()?.len(); + if length < HEADER_LEN as u64 { + return Err(invalid_data("sorted-index header is truncated")); + } + let mut input = BufReader::new(file); + let mut encoded = [0; HEADER_LEN]; + input.read_exact(&mut encoded)?; + let identity = decode_header_identity(encoded)?; + let remaining_bytes = length - HEADER_LEN as u64; + let min_entry_len = (ENTRY_PREFIX_LEN + ENTRY_FIXED_BODY_LEN) as u64; + if identity.count > remaining_bytes / min_entry_len { + return Err(invalid_data("sorted-index count exceeds remaining file bytes")); + } + let mut reader = Reader::::from_reader(input, remaining_bytes, identity.count, false); + let count = usize::try_from(identity.count).map_err(invalid_integer)?; + let mut entries = Vec::with_capacity(count); + while let Some(entry) = reader.read_next_borrowed()? { + entries.push((entry.sort_key, entry.primary_key, entry.locator)); + } + Ok(Some(Self { database_id: identity.database_id, generation: identity.generation, entries })) + } + + pub fn write_merged(mut self, index_path: &Path, changed: &[(SortKey, K)]) -> io::Result<()> { + let mut changed_map = std::collections::BTreeMap::new(); + for (sort_key, primary_key) in changed { + changed_map.insert(primary_key, sort_key); + } + if !changed_map.is_empty() { + self.entries.retain(|(_, primary_key, _)| !changed_map.contains_key(primary_key)); + let dummy_locator = Locator { leaf_page_id: 1, slot_index: 0, serialized_key_crc32: 0 }; + for (primary_key, sort_key) in changed_map { + self.entries.push((sort_key.clone(), primary_key.clone(), dummy_locator)); + } + self.entries.sort_unstable_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + } + + publish_index(index_path, self.database_id, self.generation, &self.entries) + } + } + + pub(crate) fn publish_index<'a, SortKey, K>( + destination: &Path, + database_id: [u8; 16], + generation: u64, + entries: impl IntoIterator, + ) -> io::Result<()> + where + SortKey: Serialize + 'a, + K: Serialize + 'a, + { + let temporary = temporary_path(destination)?; + let write_result = (|| -> io::Result<()> { + let mut writer = Writer::::new(&temporary, database_id, generation)?; + for (sort_key, primary_key, locator) in entries { + writer.push(sort_key, primary_key, *locator)?; + } + writer.finish()?; + Ok(()) + })(); + + if let Err(error) = write_result { + let _ = std::fs::remove_file(&temporary); + return Err(error); + } + + publish_database(&temporary, destination, sync_parent_directory).map_err(io::Error::from) + } } +pub(crate) use v4::publish_index; + #[cfg(test)] mod tests { use super::*; @@ -370,7 +512,120 @@ mod tests { assert_eq!(entry.locator, locator); assert!(reader.read_next()?.is_none()); assert!(v4::Reader::::open(&path, [0; 16], 9).is_err()); - assert!(v4::Reader::::open(&path, database_id, 10).is_err()); + assert!(v4::Reader::::open(&path, database_id, 8).is_err()); + let stale_reader = v4::Reader::::open(&path, database_id, 10)?; + assert!(stale_reader.is_stale()); + Ok(()) + } + + #[test] + fn stale_generation_by_key_read_skips_missing_keys_and_returns_sorted() -> io::Result<()> { + use crate::BPlusTreeUpdate; + + let dir = tempdir()?; + let database = dir.path().join("stale_test.db"); + let index = dir.path().join("stale_test.idx"); + + // Key 1 -> sort 30, Key 2 -> sort 10, Key 3 -> sort 20 + let mut tree = V3Tree::new(); + tree.insert(1u32, (30u32, String::from("one"))); + tree.insert(2u32, (10u32, String::from("two"))); + tree.insert(3u32, (20u32, String::from("three"))); + tree.store_with_index(&database, |v| v.0)?; + + let snapshot = v4::IndexSnapshot::::read(&index)?.expect("snapshot exists"); + + // Update database in place: modify key 2 and delete key 3, incrementing generation + let mut updater = BPlusTreeUpdate::::try_new(&database)?; + updater.upsert(&2, &(10, String::from("two-updated")))?; + updater.delete(&3)?; + updater.commit()?; + + // Restore index snapshot (written with stale generation) + snapshot.write_merged(&index, &[])?; + + // Index generation is now older than DB generation + let query = V3Query::::try_new(&database)?; + let iterator = v4::OwnedIterator::::open(query, &index)?; + let items: Vec<(u32, (u32, String))> = iterator.collect::>>()?; + + // Expected order: key 2 (sort 10), key 1 (sort 30); key 3 (deleted) is skipped + assert_eq!(items.len(), 2); + assert_eq!(items[0], (2, (10, String::from("two-updated")))); + assert_eq!(items[1], (1, (30, String::from("one")))); + + Ok(()) + } + + #[test] + fn merge_with_replacement_of_existing_primary_key_maintains_canonical_order() -> io::Result<()> { + let dir = tempdir()?; + let index = dir.path().join("merge_test.idx"); + let database_id = [0x42; 16]; + let generation = 5; + + let mut writer = v4::Writer::::new(&index, database_id, generation)?; + let dummy_loc = Locator { leaf_page_id: 1, slot_index: 0, serialized_key_crc32: 0 }; + writer.push(&10, &1, dummy_loc)?; + writer.push(&30, &3, dummy_loc)?; + writer.finish()?; + + let snapshot = v4::IndexSnapshot::::read(&index)?.expect("snapshot exists"); + snapshot.write_merged(&index, &[(5, 0), (50, 1)])?; + + let mut reader = v4::Reader::::open(&index, database_id, generation)?; + let mut entries = Vec::new(); + while let Some(entry) = reader.read_next()? { + entries.push((entry.sort_key, entry.primary_key)); + } + + assert_eq!(entries, vec![(5, 0), (30, 3), (50, 1)]); + Ok(()) + } + + #[test] + fn newer_index_generation_rejected() -> io::Result<()> { + let dir = tempdir()?; + let database = dir.path().join("newer_gen.db"); + let index = dir.path().join("newer_gen.idx"); + + let mut tree = V3Tree::new(); + tree.insert(1u32, String::from("one")); + tree.store(&database)?; + + let query = V3Query::::try_new(&database)?; + let (database_id, generation) = query.snapshot_identity(); + drop(query); + + let mut writer = v4::Writer::::new(&index, database_id, generation + 1)?; + writer.push(&1, &1, Locator { leaf_page_id: 1, slot_index: 0, serialized_key_crc32: 0 })?; + writer.finish()?; + + let query = V3Query::::try_new(&database)?; + let Err(err) = v4::OwnedIterator::::open(query, &index) else { + panic!("expected newer generation error"); + }; + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert!(err.to_string().contains("newer than tree generation")); + Ok(()) + } + + #[test] + fn database_id_mismatch_rejected() -> io::Result<()> { + let dir = tempdir()?; + let database = dir.path().join("mismatch.db"); + let index = dir.path().join("mismatch.idx"); + + let mut tree = V3Tree::new(); + tree.insert(1u32, String::from("one")); + tree.store(&database)?; + + let mut writer = v4::Writer::::new(&index, [0xaa; 16], 1)?; + writer.push(&1, &1, Locator { leaf_page_id: 1, slot_index: 0, serialized_key_crc32: 0 })?; + writer.finish()?; + + let query = V3Query::::try_new(&database)?; + assert!(v4::OwnedIterator::::open(query, &index).is_err()); Ok(()) } @@ -464,4 +719,21 @@ mod tests { assert!(iterator.next().is_none()); Ok(()) } + + #[test] + fn oversized_header_count_rejected() -> io::Result<()> { + let dir = tempdir()?; + let path = dir.path().join("oversized.idx"); + let database_id = [0x55; 16]; + let generation = 1; + let encoded = v4::header(database_id, generation, 100); + fs::write(&path, encoded)?; + + let Err(err) = v4::IndexSnapshot::::read(&path) else { + panic!("oversized count must fail"); + }; + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert!(err.to_string().contains("sorted-index count")); + Ok(()) + } } diff --git a/backend/btree/src/v3/mod.rs b/backend/btree/src/v3/mod.rs index 3d00882cd..a10f40065 100644 --- a/backend/btree/src/v3/mod.rs +++ b/backend/btree/src/v3/mod.rs @@ -7,11 +7,13 @@ mod wal; pub use format::{Locator, MAGIC, STORAGE_VERSION_V3 as STORAGE_VERSION}; pub use publish::{publish_staged_database, BPlusTreeStagingArtifacts}; +pub(crate) use tree::{publish_database, temporary_path}; #[allow(unused_imports)] pub use tree::{ BPlusTree, BPlusTreeDiskIterator, BPlusTreeDiskIteratorOwned, BPlusTreeKeyIterator, BPlusTreeQuery, BPlusTreeRangeIterator, BPlusTreeSerialWriter, BPlusTreeUpdate, FlushPolicy, }; +pub(crate) use wal::sync_parent_directory; #[derive(Clone, Debug, Eq, PartialEq)] pub enum BPlusTreeMetadata { diff --git a/backend/btree/src/v3/tree.rs b/backend/btree/src/v3/tree.rs index cfa860ffb..384f5d729 100644 --- a/backend/btree/src/v3/tree.rs +++ b/backend/btree/src/v3/tree.rs @@ -37,6 +37,7 @@ pub(crate) struct InternalSplit<'a, T> { #[cfg(test)] thread_local! { static INTERNAL_KEY_DECODE_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static VERIFY_FULL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; } #[derive(Clone, Debug)] @@ -126,6 +127,7 @@ mod query; mod serial_writer; mod transaction; mod update; +mod verified_cache; mod verify; #[cfg(test)] @@ -135,13 +137,17 @@ pub(crate) use self::mutation::{choose_internal_split, choose_leaf_split, used_l #[cfg(test)] use self::query::{internal_key_decode_count, reset_internal_key_decode_count}; #[cfg(test)] -pub(crate) use self::query::{read_leaf_value, validate_locator}; -#[cfg(test)] use self::transaction::page_byte_range; +#[cfg(test)] +pub(super) use self::verified_cache::{is_verified_with_cache, mark_verified_with_cache}; +pub(super) use self::{ + bulk_store::publish_database_and_invalidate_sorted_index, + error::PublishDatabaseError, + verified_cache::{is_verified, mark_verified, VerifiedIdentity}, +}; use self::{ bulk_store::{ - build_pages, internal_page, leaf_page, temporary_path, verify_internal_page, verify_leaf_page, PageSink, - StoredDatabase, + build_pages, internal_page, leaf_page, verify_internal_page, verify_leaf_page, PageSink, StoredDatabase, }, error::{invalid_data, invalid_input}, iterators::{record_page_visit, ChainGuard, CursorState}, @@ -154,8 +160,15 @@ use self::{ update::ActiveBatch, verify::{validate_transaction_links, VerifyLeaf, VerifyValue, Visit}, }; -pub(super) use self::{ - bulk_store::{publish_database, publish_database_and_invalidate_sorted_index}, - error::PublishDatabaseError, +pub(crate) use self::{ + bulk_store::{publish_database, temporary_path}, + query::search_leaf, + verify::verify_full, }; -pub(crate) use self::{query::search_leaf, verify::verify_full}; +#[cfg(test)] +pub(crate) use self::{ + query::{read_leaf_value, validate_locator}, + verify::{reset_verify_full_count, verify_full_count}, +}; +#[cfg(test)] +pub(crate) use super::wal::leave_committed_test_wal; diff --git a/backend/btree/src/v3/tree/bulk_store.rs b/backend/btree/src/v3/tree/bulk_store.rs index 9a897d3cf..1c8091d0d 100644 --- a/backend/btree/src/v3/tree/bulk_store.rs +++ b/backend/btree/src/v3/tree/bulk_store.rs @@ -289,12 +289,12 @@ where Ok(level.first().ok_or_else(|| invalid_input("tree has no root page"))?.page_id) } -pub(super) fn temporary_path(filepath: &Path) -> io::Result { +pub(crate) fn temporary_path(filepath: &Path) -> io::Result { let name = filepath.file_name().ok_or_else(|| invalid_input("database path has no file name"))?.to_string_lossy(); Ok(filepath.with_file_name(format!("{name}.{}.v3.tmp", uuid::Uuid::new_v4()))) } -pub(in crate::v3) fn publish_database( +pub(crate) fn publish_database( temporary: &Path, destination: &Path, sync_directory: impl FnOnce(&Path) -> io::Result<()>, diff --git a/backend/btree/src/v3/tree/error.rs b/backend/btree/src/v3/tree/error.rs index dbd09d140..4b1ce2d26 100644 --- a/backend/btree/src/v3/tree/error.rs +++ b/backend/btree/src/v3/tree/error.rs @@ -1,5 +1,5 @@ #[derive(Debug)] -pub(in crate::v3) enum PublishDatabaseError { +pub(crate) enum PublishDatabaseError { NotPublished(io::Error), PublishedDurabilityUnknown(io::Error), } diff --git a/backend/btree/src/v3/tree/memory.rs b/backend/btree/src/v3/tree/memory.rs index 8da92e89e..0971ad4d1 100644 --- a/backend/btree/src/v3/tree/memory.rs +++ b/backend/btree/src/v3/tree/memory.rs @@ -1,7 +1,8 @@ use super::{ - build_pages, invalid_data, invalid_input, publish_database, publish_database_and_invalidate_sorted_index, + build_pages, invalid_data, invalid_input, mark_verified, publish_database_and_invalidate_sorted_index, recover_pending_under_existing_lock, sync_parent_directory, temporary_path, verify_full, with_exclusive_sidecar, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, DatabaseHeader, PageSink, StoredDatabase, VerificationReport, + VerifiedIdentity, }; use serde::{Deserialize, Serialize}; use std::{ @@ -103,10 +104,11 @@ where drop(sink); let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&temp_path)?)?; let verification = verify_full(&mut query)?; + let identity = VerifiedIdentity::from_header(&query.header); drop(query); - Ok((root_page_id, verification)) + Ok((root_page_id, verification, identity)) })(); - let (root_page_id, verification) = match prepared { + let (root_page_id, verification, identity) = match prepared { Ok(prepared) => prepared, Err(error) => { let _ = std::fs::remove_file(&temp_path); @@ -114,6 +116,7 @@ where } }; publish_database_and_invalidate_sorted_index(&temp_path, filepath, sync_published_directory)?; + mark_verified(filepath, identity); self.dirty = false; Ok(StoredDatabase { root_page_id, verification }) } @@ -186,20 +189,7 @@ where drop(query); let index_path = crate::common::get_file_path_for_db_index(filepath); - let temporary = temporary_path(&index_path)?; - let prepared = (|| { - let mut writer = crate::sorted_index::v4::Writer::new(&temporary, database_id, generation)?; - for (sort_key, primary_key, locator) in &entries { - writer.push(sort_key, primary_key, *locator)?; - } - let _ = writer.finish()?; - Ok(()) - })(); - if let Err(error) = prepared { - let _ = std::fs::remove_file(&temporary); - return Err(error); - } - publish_database(&temporary, &index_path, sync_parent_directory).map_err(io::Error::from) + crate::sorted_index::publish_index(&index_path, database_id, generation, &entries) } } diff --git a/backend/btree/src/v3/tree/query.rs b/backend/btree/src/v3/tree/query.rs index ba7cb39f5..ada7a7df7 100644 --- a/backend/btree/src/v3/tree/query.rs +++ b/backend/btree/src/v3/tree/query.rs @@ -698,7 +698,7 @@ where K: Ord + for<'de> Deserialize<'de>, V: for<'de> Deserialize<'de>, { - pub(super) fn query_io(&mut self, key: &K) -> io::Result> { + pub(crate) fn query_io(&mut self, key: &K) -> io::Result> { let Some((leaf_id, range)) = self.locate_cell(key)? else { return Ok(None) }; self.decode_entry_range(leaf_id, range).map(|entry| entry.map(|(_, value)| value)) } diff --git a/backend/btree/src/v3/tree/tests.rs b/backend/btree/src/v3/tree/tests.rs index 8e4279320..77f655fbe 100644 --- a/backend/btree/src/v3/tree/tests.rs +++ b/backend/btree/src/v3/tree/tests.rs @@ -7,6 +7,7 @@ mod persistence; mod protocol; mod query; mod support; +mod verified_cache; use self::support::{ assert_writer_is_blocked, database_header, empty_leaf, finish_writer, invalid_data, invalid_input, random_value, diff --git a/backend/btree/src/v3/tree/tests/verified_cache.rs b/backend/btree/src/v3/tree/tests/verified_cache.rs new file mode 100644 index 000000000..6f13ce91f --- /dev/null +++ b/backend/btree/src/v3/tree/tests/verified_cache.rs @@ -0,0 +1,241 @@ +use super::super::{ + is_verified_with_cache, leave_committed_test_wal, mark_verified_with_cache, reset_verify_full_count, + verify_full_count, BPlusTree, BPlusTreeUpdate, DatabaseHeader, VerifiedIdentity, PAGE_SIZE, +}; +use crate::common::{read_exact_at_offset, write_all_at_offset}; +use std::{ + fs::{self, OpenOptions}, + io, +}; + +#[test] +fn second_try_new_on_unchanged_file_skips_verify_full() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("unchanged.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + reset_verify_full_count(); + let updater1 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 0); + drop(updater1); + + let updater2 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 0); + drop(updater2); + Ok(()) +} + +#[test] +fn uncached_file_runs_verify_full_once_and_subsequent_skips() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let seed_path = dir.path().join("seed.db"); + let target_path = dir.path().join("target_uncached.db"); + + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&seed_path)?; + + // Copying the file produces an uncached database at target_path + fs::copy(&seed_path, &target_path)?; + + reset_verify_full_count(); + + let updater1 = BPlusTreeUpdate::::try_new(&target_path)?; + assert_eq!(verify_full_count(), 1); + drop(updater1); + + let updater2 = BPlusTreeUpdate::::try_new(&target_path)?; + assert_eq!(verify_full_count(), 1); + drop(updater2); + Ok(()) +} + +#[test] +fn new_instance_after_commit_skips_verify_full() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("after_commit.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + let mut updater1 = BPlusTreeUpdate::::try_new(&path)?; + updater1.upsert(&2, &200)?; + updater1.commit()?; + drop(updater1); + + reset_verify_full_count(); + let updater2 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 0); + drop(updater2); + Ok(()) +} + +#[test] +fn replacement_by_store_runs_verify_full_once() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("replacement_store.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + let updater1 = BPlusTreeUpdate::::try_new(&path)?; + drop(updater1); + + reset_verify_full_count(); + let mut replacement = BPlusTree::new(); + replacement.insert(2u32, 200u64); + replacement.store(&path)?; + assert_eq!(verify_full_count(), 1); + + let updater2 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 1); + drop(updater2); + Ok(()) +} + +#[test] +fn external_write_changing_generation_triggers_verify_full() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("external_write.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + let updater1 = BPlusTreeUpdate::::try_new(&path)?; + drop(updater1); + + let file = OpenOptions::new().read(true).write(true).open(&path)?; + let mut header_bytes = [0u8; PAGE_SIZE]; + read_exact_at_offset(&file, &mut header_bytes, 0)?; + let mut header = DatabaseHeader::decode(&header_bytes)?; + header.generation = header.generation.checked_add(1).ok_or_else(|| io::Error::other("generation overflow"))?; + let encoded = header.encode()?; + write_all_at_offset(&file, &encoded, 0)?; + drop(file); + + reset_verify_full_count(); + let updater2 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 1); + drop(updater2); + + let updater3 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 1); + drop(updater3); + Ok(()) +} + +#[test] +fn wal_recovery_advancing_generation_triggers_verify_full() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("wal_recovery.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + let updater1 = BPlusTreeUpdate::::try_new(&path)?; + drop(updater1); + + leave_committed_test_wal(&path)?; + + reset_verify_full_count(); + let updater2 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 1); + drop(updater2); + + let updater3 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 1); + drop(updater3); + Ok(()) +} + +#[test] +fn corrupt_database_with_unverified_identity_is_rejected() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let seed_path = dir.path().join("seed_corrupt.db"); + let corrupt_path = dir.path().join("target_corrupt.db"); + + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.insert(2u32, 200u64); + tree.store(&seed_path)?; + + // Copying creates an uncached database at corrupt_path + fs::copy(&seed_path, &corrupt_path)?; + + // Corrupt root page bytes (page 1) + let file = OpenOptions::new().read(true).write(true).open(&corrupt_path)?; + let garbage = [0xff; 128]; + write_all_at_offset(&file, &garbage, PAGE_SIZE as u64)?; + drop(file); + + reset_verify_full_count(); + let result = BPlusTreeUpdate::::try_new(&corrupt_path); + assert!(result.is_err()); + assert_eq!(verify_full_count(), 1); + Ok(()) +} + +#[test] +fn compact_marks_resulting_identity_and_subsequent_try_new_skips() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("compact.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.upsert(&2, &200)?; + updater.compact()?; + drop(updater); + + reset_verify_full_count(); + let updater2 = BPlusTreeUpdate::::try_new(&path)?; + assert_eq!(verify_full_count(), 0); + drop(updater2); + Ok(()) +} + +#[test] +fn ensure_transaction_skips_when_committed_by_another_instance() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("mismatch_ensure.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, 100u64); + tree.store(&path)?; + + let mut updater1 = BPlusTreeUpdate::::try_new(&path)?; + let mut updater2 = BPlusTreeUpdate::::try_new(&path)?; + + updater1.upsert(&2, &200)?; + updater1.commit()?; + + reset_verify_full_count(); + updater2.upsert(&3, &300)?; + assert_eq!(verify_full_count(), 0); + updater2.commit()?; + Ok(()) +} + +#[test] +fn poisoned_mutex_treats_as_not_verified_and_never_panics() { + use std::{collections::HashMap, sync::Mutex}; + + let cache = Mutex::new(HashMap::new()); + let _ = std::panic::catch_unwind(|| { + let _guard = cache.lock().unwrap(); + panic!("test panic to poison cache mutex"); + }); + assert!(cache.is_poisoned()); + + let identity = VerifiedIdentity::new([0; 16], 1, 1); + let path = std::path::Path::new("/test"); + + // Poisoned mutex must be treated as not verified + assert!(!is_verified_with_cache(&cache, path, identity)); + + // Marking on poisoned mutex must never panic and do nothing + mark_verified_with_cache(&cache, path, identity); + assert!(!is_verified_with_cache(&cache, path, identity)); +} diff --git a/backend/btree/src/v3/tree/update.rs b/backend/btree/src/v3/tree/update.rs index 74382de57..67441f5dd 100644 --- a/backend/btree/src/v3/tree/update.rs +++ b/backend/btree/src/v3/tree/update.rs @@ -1,9 +1,9 @@ use super::{ - commit_ordered_page_refs_under_existing_lock, encode_value, invalid_data, invalid_input, query_transaction, - recover_pending_under_existing_lock, stage_delete, stage_upsert, validate_transaction_links, verify_full, - with_exclusive_sidecar, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, BPlusTreeUpdate, DatabaseHeader, - DatabaseImage, ExclusiveSidecarGuard, FlushPolicy, WalOperationError, WalOutcome, WriteScratch, WriteTransaction, - PAGE_SIZE, + commit_ordered_page_refs_under_existing_lock, encode_value, invalid_data, invalid_input, is_verified, + mark_verified, query_transaction, recover_pending_under_existing_lock, stage_delete, stage_upsert, + validate_transaction_links, verify_full, with_exclusive_sidecar, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, + BPlusTreeUpdate, DatabaseHeader, DatabaseImage, ExclusiveSidecarGuard, FlushPolicy, VerifiedIdentity, + WalOperationError, WalOutcome, WriteScratch, WriteTransaction, PAGE_SIZE, }; use crate::{ codec::{binary_serialize, binary_serialize_into}, @@ -18,6 +18,18 @@ pub(super) struct ActiveBatch { pub(super) _guard: ExclusiveSidecarGuard, } +fn verify_if_needed(filepath: &Path, query: &mut BPlusTreeQuery) -> io::Result +where + K: Ord + for<'de> Deserialize<'de> + Clone, +{ + let identity = VerifiedIdentity::from_header(&query.header); + if !is_verified(filepath, identity) { + let _ = verify_full(query)?; + mark_verified(filepath, identity); + } + Ok(identity) +} + impl BPlusTreeUpdate where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, @@ -25,16 +37,13 @@ where { pub fn try_new(filepath: &Path) -> io::Result { let mut query = BPlusTreeQuery::::try_new(filepath)?; - let _ = verify_full(&mut query)?; - let database_id = query.header.database_id; - let verified_generation = query.header.generation; - let verified_next_page_id = query.header.next_page_id; + let identity = verify_if_needed(filepath, &mut query)?; drop(query); Ok(Self { filepath: filepath.to_path_buf(), - database_id, - verified_generation, - verified_next_page_id, + database_id: identity.database_id, + verified_generation: identity.generation, + verified_next_page_id: identity.next_page_id, flush_policy: FlushPolicy::Immediate, active: None, scratch: WriteScratch::default(), @@ -62,10 +71,10 @@ where || header.next_page_id != self.verified_next_page_id { let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&self.filepath)?)?; - let _ = verify_full(&mut query)?; - self.database_id = query.header.database_id; - self.verified_generation = query.header.generation; - self.verified_next_page_id = query.header.next_page_id; + let identity = verify_if_needed(&self.filepath, &mut query)?; + self.database_id = identity.database_id; + self.verified_generation = identity.generation; + self.verified_next_page_id = identity.next_page_id; } let transaction = WriteTransaction::new(header, base.as_slice().len())?; self.active = Some(ActiveBatch { base, transaction, _guard: guard }); @@ -339,11 +348,13 @@ where let committed_header = DatabaseHeader::decode(prepared[0].1)?; let committed_generation = committed_header.generation; let committed_next_page_id = committed_header.next_page_id; + let committed_identity = VerifiedIdentity::from_header(&committed_header); let result = commit_ordered_page_refs_under_existing_lock(&self.filepath, &prepared); match result { Ok(()) => { self.verified_generation = committed_generation; self.verified_next_page_id = committed_next_page_id; + mark_verified(&self.filepath, committed_identity); Ok(()) } Err(error) => { @@ -354,6 +365,7 @@ where if outcome == Some(WalOutcome::CommittedCleanupPending) { self.verified_generation = committed_generation; self.verified_next_page_id = committed_next_page_id; + mark_verified(&self.filepath, committed_identity); } if let Err(recovery_error) = recover_pending_under_existing_lock(&self.filepath) { log::error!( diff --git a/backend/btree/src/v3/tree/verified_cache.rs b/backend/btree/src/v3/tree/verified_cache.rs new file mode 100644 index 000000000..7fd87768f --- /dev/null +++ b/backend/btree/src/v3/tree/verified_cache.rs @@ -0,0 +1,66 @@ +use super::DatabaseHeader; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::{LazyLock, Mutex}, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct VerifiedIdentity { + pub(crate) database_id: [u8; 16], + pub(crate) generation: u64, + pub(crate) next_page_id: u64, +} + +impl VerifiedIdentity { + #[cfg(test)] + pub(crate) const fn new(database_id: [u8; 16], generation: u64, next_page_id: u64) -> Self { + Self { database_id, generation, next_page_id } + } + + pub(crate) const fn from_header(header: &DatabaseHeader) -> Self { + Self { database_id: header.database_id, generation: header.generation, next_page_id: header.next_page_id } + } +} + +static VERIFIED_IDENTITIES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +fn canonical_path(path: &Path) -> PathBuf { + match std::fs::canonicalize(path) { + Ok(canonical) => canonical, + Err(_) => path.to_path_buf(), + } +} + +pub(crate) fn is_verified_with_cache( + cache: &Mutex>, + path: &Path, + identity: VerifiedIdentity, +) -> bool { + let canonical = canonical_path(path); + let Ok(guard) = cache.lock() else { + return false; + }; + guard.get(&canonical).copied() == Some(identity) +} + +pub(crate) fn mark_verified_with_cache( + cache: &Mutex>, + path: &Path, + identity: VerifiedIdentity, +) { + let canonical = canonical_path(path); + let Ok(mut guard) = cache.lock() else { + return; + }; + let _ = guard.insert(canonical, identity); +} + +pub(crate) fn is_verified(path: &Path, identity: VerifiedIdentity) -> bool { + is_verified_with_cache(&VERIFIED_IDENTITIES, path, identity) +} + +pub(crate) fn mark_verified(path: &Path, identity: VerifiedIdentity) { + mark_verified_with_cache(&VERIFIED_IDENTITIES, path, identity); +} diff --git a/backend/btree/src/v3/tree/verify.rs b/backend/btree/src/v3/tree/verify.rs index 78e4462bd..18192520f 100644 --- a/backend/btree/src/v3/tree/verify.rs +++ b/backend/btree/src/v3/tree/verify.rs @@ -1,3 +1,5 @@ +#[cfg(test)] +use super::VERIFY_FULL_COUNT; use super::{ decode_key, invalid_data, minimum_tree_key, push_child_visits, record_page_visit, validate_leaf_backlink, validated_overflow_chain_pages, verify_internal_page, verify_leaf_page, BPlusTreeQuery, Compression, @@ -233,10 +235,19 @@ fn verify_page_ownership( Ok(()) } +#[cfg(test)] +pub(crate) fn reset_verify_full_count() { VERIFY_FULL_COUNT.set(0); } + +#[cfg(test)] +pub(crate) fn verify_full_count() -> usize { VERIFY_FULL_COUNT.get() } + pub(crate) fn verify_full(query: &mut BPlusTreeQuery) -> io::Result where K: Ord + for<'de> Deserialize<'de> + Clone, { + #[cfg(test)] + VERIFY_FULL_COUNT.set(VERIFY_FULL_COUNT.get().saturating_add(1)); + let mut tree_pages = HashSet::new(); let mut active = HashSet::new(); let mut overflow_pages = HashSet::new(); diff --git a/backend/btree/src/v3/wal.rs b/backend/btree/src/v3/wal.rs index a3bf97d71..21ac33bd1 100644 --- a/backend/btree/src/v3/wal.rs +++ b/backend/btree/src/v3/wal.rs @@ -547,14 +547,14 @@ fn new_transaction_id() -> io::Result { } #[cfg(unix)] -pub(super) fn sync_parent_directory(path: &Path) -> io::Result<()> { +pub(crate) fn sync_parent_directory(path: &Path) -> io::Result<()> { let parent = path.parent().filter(|parent| !parent.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")); File::open(parent)?.sync_all() } #[cfg(not(unix))] #[allow(clippy::unnecessary_wraps)] -pub(super) fn sync_parent_directory(_path: &Path) -> io::Result<()> { Ok(()) } +pub(crate) fn sync_parent_directory(_path: &Path) -> io::Result<()> { Ok(()) } pub(crate) fn invalidate_sorted_index(database: &Path) -> io::Result<()> { let index = crate::common::get_file_path_for_db_index(database); @@ -763,6 +763,40 @@ pub(super) fn leave_uncommitted_test_wal_after_database_write(database: &Path) - } } +#[cfg(test)] +pub(crate) fn leave_committed_test_wal(database: &Path) -> io::Result<()> { + let mut header_page = [0u8; PAGE_SIZE]; + File::open(database)?.read_exact(&mut header_page)?; + let mut updated_header = DatabaseHeader::decode(&header_page)?; + updated_header.generation = updated_header + .generation + .checked_add(1) + .ok_or_else(|| io::Error::other("test database generation overflow"))?; + let prepared = [(0, updated_header.encode()?)]; + let commit_result = commit_prepared_pages_with_hook(database, &prepared, |boundary| { + if boundary == CommitBoundary::CommitSynced { + Err(io::Error::other("test fault after commit sync")) + } else { + Ok(()) + } + }); + match commit_result { + Err(error) if !error.to_string().contains("test fault after commit sync") => Err(io::Error::new( + error.kind(), + format!("test commit for {} failed before the requested boundary: {error}", database.display()), + )), + Err(error) if !wal_path(database).try_exists()? => Err(io::Error::new( + error.kind(), + format!( + "test commit for {} reached the requested boundary without leaving an active WAL: {error}", + database.display() + ), + )), + Err(_) => Ok(()), + Ok(()) => Err(io::Error::other(format!("test commit for {} unexpectedly completed", database.display()))), + } +} + #[cfg(test)] fn commit_prepared_pages_with_hook_under_existing_lock( database_path: &Path, diff --git a/backend/core/src/utils/network/request.rs b/backend/core/src/utils/network/request.rs index b12bb2456..14db8cebb 100644 --- a/backend/core/src/utils/network/request.rs +++ b/backend/core/src/utils/network/request.rs @@ -76,6 +76,7 @@ const RESOURCE_DESTINATION_LOOKUP_TIMEOUT: Duration = Duration::from_millis(250) pub struct RequestFetchOptions { stop_on_proxy_authentication_required: bool, pub attempt_idle_timeout: Option, + header_timeout_only: bool, content_coding: OutboundContentCodingPolicy, resource_retry: ResourceRetryExecution, return_http_errors: bool, diff --git a/backend/core/src/utils/network/request/download.rs b/backend/core/src/utils/network/request/download.rs index 7ac427835..c30ed0fdc 100644 --- a/backend/core/src/utils/network/request/download.rs +++ b/backend/core/src/utils/network/request/download.rs @@ -55,6 +55,11 @@ impl RequestFetchOptions { Self { attempt_idle_timeout: Some(timeout.max(Duration::from_millis(1))), ..Self::default() } } + pub const fn with_header_timeout_only(mut self) -> Self { + self.header_timeout_only = true; + self + } + pub const fn with_content_coding(mut self, content_coding: OutboundContentCodingPolicy) -> Self { self.content_coding = content_coding; self diff --git a/backend/core/src/utils/network/request/options.rs b/backend/core/src/utils/network/request/options.rs index 350627d82..ce9a86951 100644 --- a/backend/core/src/utils/network/request/options.rs +++ b/backend/core/src/utils/network/request/options.rs @@ -9,8 +9,10 @@ pub(super) enum ResourceRetryExecution { } pub(super) fn apply_request_fetch_options(request: &mut reqwest::Request, options: RequestFetchOptions) { - if let Some(timeout) = options.attempt_idle_timeout { - *request.timeout_mut() = Some(timeout); + if !options.header_timeout_only { + if let Some(timeout) = options.attempt_idle_timeout { + *request.timeout_mut() = Some(timeout); + } } apply_outbound_content_coding_policy(request.headers_mut(), options.content_coding); } diff --git a/backend/curation/src/kernel.rs b/backend/curation/src/kernel.rs index 6c6bc8a27..f7c2042f0 100644 --- a/backend/curation/src/kernel.rs +++ b/backend/curation/src/kernel.rs @@ -135,7 +135,7 @@ impl CurationMediaScope { fn includes_playlist_item(self, item_type: PlaylistItemType) -> bool { match self { Self::Movies => item_type.is_video(), - Self::Series => matches!(item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo), + Self::Series => item_type.is_series_info(), Self::Both => matches!( item_type, PlaylistItemType::Video @@ -479,8 +479,7 @@ fn create_category_from_items( for item in items { let projected_item = clone_item_for_category(item, specification, &group_title); let parent_uuid = projected_item.header.uuid.intern(); - let is_series_info = - matches!(projected_item.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo); + let is_series_info = projected_item.header.item_type.is_series_info(); let child_lookup_keys = if is_series_info { series_info_child_lookup_keys(item) } else { Vec::new() }; let cluster = projected_item.header.xtream_cluster; matched_items_by_cluster.entry(cluster).or_default().push(projected_item); diff --git a/backend/iptv/src/xtream/mod.rs b/backend/iptv/src/xtream/mod.rs index c4447c929..2ea536541 100644 --- a/backend/iptv/src/xtream/mod.rs +++ b/backend/iptv/src/xtream/mod.rs @@ -134,12 +134,15 @@ pub async fn get_xtream_stream_info( vod_pli.additional_properties = Some(StreamProperties::Video(Box::new(video_stream_props))); - if let Err(err) = write_playlist_item_update(app_config, &target.name, &vod_pli).await { - error!("Failed to persist video stream: {err}"); - } - - if target.use_memory_cache { - playlists.update_playlist_items(target, vec![&vod_pli]).await; + match write_playlist_item_update(app_config, &target.name, &vod_pli).await { + Ok(()) => { + if target.use_memory_cache { + playlists.update_playlist_items(target, &[&vod_pli]).await; + } + } + Err(err) => { + error!("Failed to persist video stream: {err}"); + } } if let Some(value) = xtream_resolve_stream_info( @@ -258,7 +261,7 @@ pub async fn get_xtream_stream_info( let xtream_episodes: Vec = episodes.iter().map(XtreamPlaylistItem::from).collect(); - if let Err(err) = write_playlist_batch_item_upsert( + match write_playlist_batch_item_upsert( app_config, &target.name, XtreamCluster::Series, @@ -266,12 +269,15 @@ pub async fn get_xtream_stream_info( ) .await { - error!("Failed to persist playlist batch item update: {err}"); - } - - if target.use_memory_cache && !in_memory_updates.is_empty() { - playlists.insert_playlist_items(target, episodes).await; - playlists.update_target_id_mapping(target, in_memory_updates).await; + Ok(()) => { + if target.use_memory_cache && !in_memory_updates.is_empty() { + playlists.insert_playlist_items(target, episodes).await; + playlists.update_target_id_mapping(target, in_memory_updates).await; + } + } + Err(err) => { + error!("Failed to persist playlist batch item update: {err}"); + } } if !provider_series.is_empty() { @@ -282,12 +288,18 @@ pub async fn get_xtream_stream_info( &mut series_pli, &provider_series, ); - if let Err(err) = - write_playlist_item_update(app_config, &target.name, &series_pli).await + match write_playlist_item_update(app_config, &target.name, &series_pli) + .await { - error!("Failed to persist series stream: {err}"); + Ok(()) => { + if target.use_memory_cache { + playlists.update_playlist_items(target, &[&series_pli]).await; + } + } + Err(err) => { + error!("Failed to persist series stream: {err}"); + } } - playlists.update_playlist_items(target, vec![&series_pli]).await; if let Some(value) = xtream_resolve_stream_info( app_config, diff --git a/backend/metadata/src/manager.rs b/backend/metadata/src/manager.rs index b81ae5975..d99f3e2f3 100644 --- a/backend/metadata/src/manager.rs +++ b/backend/metadata/src/manager.rs @@ -192,7 +192,7 @@ macro_rules! apply_cascade_updates { } if target.use_memory_cache { - Self::update_memory_cache(ctx, target_name, $cluster, updates).await; + Self::update_memory_cache(&ctx.playlists, target_name, $cluster, updates).await; } } }; diff --git a/backend/metadata/src/manager/publication.rs b/backend/metadata/src/manager/publication.rs index e6720e815..9c1699b3a 100644 --- a/backend/metadata/src/manager/publication.rs +++ b/backend/metadata/src/manager/publication.rs @@ -16,7 +16,7 @@ use tuliprox_core::{ }; use tuliprox_repository::{ get_input_storage_path, get_target_id_mapping_file, persist_input_live_info_batch, persist_input_series_info_batch, - persist_input_vod_info_batch, xtream_get_file_path, BPlusTreeQuery, TargetIdMapping, + persist_input_vod_info_batch, xtream_get_file_path, BPlusTreeQuery, PlaylistStorageState, TargetIdMapping, }; pub(super) struct DbHandle { @@ -238,22 +238,17 @@ impl InputWorker { resolved } - pub(super) async fn update_memory_cache( - ctx: &MetadataUpdateCtx, + pub(super) async fn update_memory_cache( + playlists: &PlaylistStorageState, target_name: &str, cluster: XtreamCluster, updates: Vec, ) { - let mut playlists = ctx.playlists.data.write().await; + let mut playlists = playlists.data.write().await; if let Some(playlist) = playlists.get_mut(target_name) { if let Some(xtream_storage) = &mut playlist.xtream { - let storage = match cluster { - XtreamCluster::Live => &mut xtream_storage.live, - XtreamCluster::Video => &mut xtream_storage.vod, - XtreamCluster::Series => &mut xtream_storage.series, - }; for item in updates { - storage.insert(item.virtual_id.get(), item); + xtream_storage.upsert(cluster, item); } } } diff --git a/backend/processing/src/processor/playlist/target.rs b/backend/processing/src/processor/playlist/target.rs index 4e584f949..d2b0776fc 100644 --- a/backend/processing/src/processor/playlist/target.rs +++ b/backend/processing/src/processor/playlist/target.rs @@ -1007,9 +1007,7 @@ fn item_subject_uuid(item: &PlaylistItem) -> UUIDType { fn selected_series_parent_keys(playlist: &[PlaylistGroup], selected_subjects: &HashSet) -> HashSet> { let mut parent_keys = HashSet::new(); for item in playlist.iter().flat_map(|group| &group.channels) { - if !matches!(item.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) - || !selected_subjects.contains(&item_subject_uuid(item)) - { + if !item.header.item_type.is_series_info() || !selected_subjects.contains(&item_subject_uuid(item)) { continue; } parent_keys.insert(item.get_uuid().intern()); diff --git a/backend/repository/src/lineup_guide_index.rs b/backend/repository/src/lineup_guide_index.rs index 4f39477dd..5954b5106 100644 --- a/backend/repository/src/lineup_guide_index.rs +++ b/backend/repository/src/lineup_guide_index.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use shared::{ concat_string, error::TuliproxError, - model::{PlaylistGroup, PlaylistItemType, TargetType, XtreamCluster}, + model::{PlaylistGroup, TargetType, XtreamCluster}, }; use std::{ io, @@ -176,9 +176,7 @@ fn playlist_rows(playlist: &[PlaylistGroup], view: LineupGuideView) -> Vec { - !matches!(header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) - } + LineupGuideView::M3u => !header.item_type.is_series_info(), LineupGuideView::Xtream => matches!(cluster, XtreamCluster::Live | XtreamCluster::Video), }; if !indexed { @@ -393,7 +391,7 @@ pub async fn lineup_guide_candidates( mod tests { use super::*; use shared::{ - model::{PlaylistItem, PlaylistItemHeader, VirtualId}, + model::{PlaylistItem, PlaylistItemHeader, PlaylistItemType, VirtualId}, utils::Internable, }; use tempfile::TempDir; diff --git a/backend/repository/src/m3u_repository.rs b/backend/repository/src/m3u_repository.rs index 62ab6ccf8..a8624e953 100644 --- a/backend/repository/src/m3u_repository.rs +++ b/backend/repository/src/m3u_repository.rs @@ -15,10 +15,7 @@ use log::{error, warn}; use shared::{ concat_string, error::{str_to_io_error, string_to_io_error, TuliproxError}, - model::{ - LiveStreamProperties, M3uPlaylistItem, PlaylistGroup, PlaylistItem, PlaylistItemType, StreamProperties, - XtreamCluster, - }, + model::{LiveStreamProperties, M3uPlaylistItem, PlaylistGroup, PlaylistItem, StreamProperties, XtreamCluster}, utils::{m3u_flussonic_live_file_index_key, m3u_stream_url_identity, PROVIDER_SCHEME_PREFIX}, }; use std::{ @@ -277,9 +274,7 @@ pub async fn m3u_write_playlist( let m3u_playlist = new_playlist .iter() .flat_map(|pg| &pg.channels) - .filter(|&pli| { - !matches!(pli.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) - }) + .filter(|&pli| !pli.header.item_type.is_series_info()) .map(M3uPlaylistItem::from) .collect::>(); diff --git a/backend/repository/src/playlist_mem_cache.rs b/backend/repository/src/playlist_mem_cache.rs index d0d64644e..c494f4fe8 100644 --- a/backend/repository/src/playlist_mem_cache.rs +++ b/backend/repository/src/playlist_mem_cache.rs @@ -1,13 +1,118 @@ use super::{BPlusTree, VirtualIdRecord}; -use shared::model::{M3uPlaylistItem, PlaylistItem, VirtualId, XtreamCluster, XtreamPlaylistItem}; +use shared::model::{M3uPlaylistItem, PlaylistItem, PlaylistItemType, VirtualId, XtreamCluster, XtreamPlaylistItem}; use std::collections::HashMap; use tokio::sync::RwLock; use tuliprox_core::model::ConfigTarget; +#[inline] +pub(crate) fn is_cluster_order_eligible(cluster: XtreamCluster, item_type: PlaylistItemType) -> bool { + match cluster { + XtreamCluster::Series => item_type.is_series_info(), + XtreamCluster::Live | XtreamCluster::Video => true, + } +} + pub struct PlaylistXtreamStorage { pub live: BPlusTree, + pub live_order: Vec<(u32, u32)>, pub vod: BPlusTree, + pub vod_order: Vec<(u32, u32)>, pub series: BPlusTree, + pub series_order: Vec<(u32, u32)>, +} + +impl Default for PlaylistXtreamStorage { + fn default() -> Self { Self::new() } +} + +impl PlaylistXtreamStorage { + pub fn new() -> Self { + Self { + live: BPlusTree::new(), + live_order: Vec::new(), + vod: BPlusTree::new(), + vod_order: Vec::new(), + series: BPlusTree::new(), + series_order: Vec::new(), + } + } + + pub fn from_trees( + live: BPlusTree, + vod: BPlusTree, + series: BPlusTree, + ) -> Self { + let live_order = Self::build_cluster_order(XtreamCluster::Live, &live); + let vod_order = Self::build_cluster_order(XtreamCluster::Video, &vod); + let series_order = Self::build_cluster_order(XtreamCluster::Series, &series); + Self { live, live_order, vod, vod_order, series, series_order } + } + + pub(crate) fn build_cluster_order( + cluster: XtreamCluster, + tree: &BPlusTree, + ) -> Vec<(u32, u32)> { + let mut order: Vec<(u32, u32)> = tree + .iter() + .filter(|(_, item)| is_cluster_order_eligible(cluster, item.item_type)) + .map(|(key, item)| (item.source_ordinal, *key)) + .collect(); + order.sort_unstable(); + order + } + + #[inline] + pub fn get_cluster(&self, cluster: XtreamCluster) -> (&BPlusTree, &[(u32, u32)]) { + match cluster { + XtreamCluster::Live => (&self.live, &self.live_order), + XtreamCluster::Video => (&self.vod, &self.vod_order), + XtreamCluster::Series => (&self.series, &self.series_order), + } + } + + #[inline] + fn get_cluster_mut( + &mut self, + cluster: XtreamCluster, + ) -> (&mut BPlusTree, &mut Vec<(u32, u32)>) { + match cluster { + XtreamCluster::Live => (&mut self.live, &mut self.live_order), + XtreamCluster::Video => (&mut self.vod, &mut self.vod_order), + XtreamCluster::Series => (&mut self.series, &mut self.series_order), + } + } + + /// Inserts or replaces one item. New keys are binary-search inserted into the cluster order, + /// and repositioned when `source_ordinal` changes. Series items are stored without season/episode details; + /// lookups that need them read the item from disk. In the Series cluster only `SeriesInfo`/`LocalSeriesInfo` + /// enter the order. + pub fn upsert(&mut self, cluster: XtreamCluster, mut item: XtreamPlaylistItem) { + if cluster == XtreamCluster::Series { + item.clear_series_details(); + } + let key = item.virtual_id.get(); + let new_ordinal = item.source_ordinal; + let new_eligible = is_cluster_order_eligible(cluster, item.item_type); + + let (tree, order) = self.get_cluster_mut(cluster); + let mut already_in_place = false; + if let Some(old) = tree.query(&key) { + let old_ordinal = old.source_ordinal; + let old_eligible = is_cluster_order_eligible(cluster, old.item_type); + if old_eligible { + if new_eligible && old_ordinal == new_ordinal { + already_in_place = true; + } else if let Ok(pos) = order.binary_search(&(old_ordinal, key)) { + order.remove(pos); + } + } + } + if new_eligible && !already_in_place { + let pos = order.binary_search(&(new_ordinal, key)).unwrap_or_else(|pos| pos); + order.insert(pos, (new_ordinal, key)); + } + tree.insert(key, item); + } } pub type PlaylistM3uStorage = BPlusTree; @@ -53,17 +158,12 @@ impl PlaylistStorageState { } } - pub async fn update_playlist_items(&self, target: &ConfigTarget, pli_list: Vec<&XtreamPlaylistItem>) { + pub async fn update_playlist_items(&self, target: &ConfigTarget, pli_list: &[&XtreamPlaylistItem]) { if target.use_memory_cache { if let Some(storage) = self.data.write().await.get_mut(&target.name) { if let Some(xtream) = storage.xtream.as_mut() { - for pli in pli_list { - match pli.xtream_cluster { - XtreamCluster::Live => &mut xtream.live, - XtreamCluster::Video => &mut xtream.vod, - XtreamCluster::Series => &mut xtream.series, - } - .insert(pli.virtual_id.get(), pli.clone()); + for &pli in pli_list { + xtream.upsert(pli.xtream_cluster, pli.clone()); } } } @@ -75,12 +175,7 @@ impl PlaylistStorageState { if let Some(storage) = self.data.write().await.get_mut(&target.name) { if let Some(xtream) = storage.xtream.as_mut() { for pli in pli_list { - match pli.header.xtream_cluster { - XtreamCluster::Live => &mut xtream.live, - XtreamCluster::Video => &mut xtream.vod, - XtreamCluster::Series => &mut xtream.series, - } - .insert(pli.header.virtual_id.get(), XtreamPlaylistItem::from(&pli)); + xtream.upsert(pli.header.xtream_cluster, XtreamPlaylistItem::from(&pli)); } } } @@ -134,3 +229,172 @@ impl PlaylistStorageState { } } } + +#[cfg(test)] +mod tests { + use super::*; + use shared::{ + model::{PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailProperties, StreamProperties, VirtualId}, + utils::Internable, + }; + use tuliprox_core::model::{ConfigTarget, TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput}; + + fn sample_target(use_memory_cache: bool) -> ConfigTarget { + let mut target = crate::test_config::target_with_output( + "test-target", + TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, filter: None }), + ); + target.use_memory_cache = use_memory_cache; + target + } + + fn sample_xtream_item(id: u32, cluster: XtreamCluster) -> XtreamPlaylistItem { + XtreamPlaylistItem { + virtual_id: VirtualId::new(id), + provider_id: id, + name: format!("Item {id}").intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "".intern(), + epg_channel_id: None, + xtream_cluster: cluster, + additional_properties: None, + item_type: match cluster { + XtreamCluster::Live => PlaylistItemType::Live, + XtreamCluster::Video => PlaylistItemType::Video, + XtreamCluster::Series => PlaylistItemType::SeriesInfo, + }, + category_id: 1, + input_name: "test".intern(), + channel_no: id, + source_ordinal: id, + input_stream_id: id.to_string().intern(), + upstream_user_agent: None, + } + } + + #[tokio::test] + async fn update_playlist_items_appends_only_new_keys_to_order() { + let playlists = PlaylistStorageState::new(); + let target = sample_target(true); + + // Prepopulate storage with items 10 and 20 + let mut storage = PlaylistXtreamStorage::new(); + let item10 = sample_xtream_item(10, XtreamCluster::Live); + let item20 = sample_xtream_item(20, XtreamCluster::Live); + storage.live.insert(10, item10.clone()); + storage.live.insert(20, item20.clone()); + storage.live_order = vec![(10, 10), (20, 20)]; + + playlists.cache_playlist(&target.name, PlaylistStorage::XtreamPlaylist(Box::new(storage))).await; + + // 1. Update existing key 20 with same source_ordinal: order must NOT duplicate 20 + playlists.update_playlist_items(&target, &[&item20]).await; + { + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + assert_eq!(xtream.live_order, vec![(10, 10), (20, 20)]); + } + + // 2. Add new key 30 via update_playlist_items: order must binary-search insert 30 + let item30 = sample_xtream_item(30, XtreamCluster::Live); + playlists.update_playlist_items(&target, &[&item30]).await; + { + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + assert_eq!(xtream.live_order, vec![(10, 10), (20, 20), (30, 30)]); + } + + // 3. Update existing key 20 with changed source_ordinal 5: repositions before 10 + let mut item20_moved = item20.clone(); + item20_moved.source_ordinal = 5; + playlists.update_playlist_items(&target, &[&item20_moved]).await; + { + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + assert_eq!(xtream.live_order, vec![(5, 20), (10, 10), (30, 30)]); + } + + // 4. Add existing key 10 via insert_playlist_items: order unchanged + let pli10 = PlaylistItem { + header: PlaylistItemHeader { + virtual_id: VirtualId::new(10), + xtream_cluster: XtreamCluster::Live, + source_ordinal: 10, + ..Default::default() + }, + }; + playlists.insert_playlist_items(&target, vec![pli10]).await; + { + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + assert_eq!(xtream.live_order, vec![(5, 20), (10, 10), (30, 30)]); + } + + // 5. Add new key 40 via insert_playlist_items: order must binary-search insert 40 + let pli40 = PlaylistItem { + header: PlaylistItemHeader { + virtual_id: VirtualId::new(40), + xtream_cluster: XtreamCluster::Live, + source_ordinal: 40, + ..Default::default() + }, + }; + playlists.insert_playlist_items(&target, vec![pli40]).await; + { + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + assert_eq!(xtream.live_order, vec![(5, 20), (10, 10), (30, 30), (40, 40)]); + } + } + + #[tokio::test] + async fn series_episodes_do_not_enter_series_order() { + let playlists = PlaylistStorageState::new(); + let target = sample_target(true); + + playlists.cache_playlist(&target.name, PlaylistStorage::XtreamPlaylist(Box::default())).await; + + // SeriesInfo enters the order + let series_item = crate::test_config::xtream_series_item(100, None); + playlists.update_playlist_items(&target, &[&series_item]).await; + + // Series episode does not enter the order + let mut episode_item = sample_xtream_item(200, XtreamCluster::Series); + episode_item.item_type = PlaylistItemType::Series; + playlists.update_playlist_items(&target, &[&episode_item]).await; + + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + assert_eq!(xtream.series_order, vec![(0, 100)]); + assert!(xtream.series.query(&200).is_some()); + } + + #[tokio::test] + async fn update_playlist_items_strips_series_details_in_cache() { + let playlists = PlaylistStorageState::new(); + let target = sample_target(true); + + playlists.cache_playlist(&target.name, PlaylistStorage::XtreamPlaylist(Box::default())).await; + + let details = SeriesStreamDetailProperties { year: Some(2024), seasons: None, episodes: None }; + let series_item = crate::test_config::xtream_series_item(77, Some(details)); + assert!(series_item.has_details()); + + // Update with series item carrying details + playlists.update_playlist_items(&target, &[&series_item]).await; + + let data = playlists.data.read().await; + let xtream = data.get(&target.name).unwrap().xtream.as_ref().unwrap(); + let cached = xtream.series.query(&77).expect("series item present in cache"); + assert!(!cached.has_details(), "cached series item must not have details"); + match &cached.additional_properties { + Some(StreamProperties::Series(props)) => assert_eq!(props.details, None), + other => panic!("unexpected properties: {other:?}"), + } + } +} diff --git a/backend/repository/src/playlist_repository/target.rs b/backend/repository/src/playlist_repository/target.rs index b97d5fdc1..f0805b9d5 100644 --- a/backend/repository/src/playlist_repository/target.rs +++ b/backend/repository/src/playlist_repository/target.rs @@ -2,8 +2,8 @@ use super::publication::TargetCacheReloadStage; use super::{publication::TargetPersistenceMode, PlaylistM3uStorage, PlaylistXtreamStorage, TargetPlaylistStorage}; use crate::{ - get_target_id_mapping_file, get_target_storage_path, m3u_get_file_path_for_db, xtream_get_file_path, - xtream_get_storage_path, BPlusTree, TargetIdMapping, VirtualIdRecord, + get_file_path_for_db_index, get_target_id_mapping_file, get_target_storage_path, m3u_get_file_path_for_db, + xtream_get_file_path, xtream_get_storage_path, BPlusTree, TargetIdMapping, VirtualIdRecord, }; use log::{debug, warn}; use shared::{ @@ -49,26 +49,43 @@ async fn load_target_id_mapping_as_tree( .map_err(|err| TuliproxError::Config(format!("Could not find path for target {target_name} err:{err}"))) } +type XtreamTreeAndOrder = (BPlusTree, Vec<(u32, u32)>); + +fn read_tree_and_order( + path: &Path, + index_path: &Path, + cluster: XtreamCluster, +) -> Result { + let mut tree = BPlusTree::new(); + crate::visit_xtream_cluster_entries(path, index_path, cluster, |key, item| { + tree.insert(key, item); + true + })?; + let order = PlaylistXtreamStorage::build_cluster_order(cluster, &tree); + Ok((tree, order)) +} + async fn load_xtream_playlist_as_tree( app_config: &AppConfig, storage_path: &Path, cluster: XtreamCluster, -) -> Result, TuliproxError> { +) -> Result<(BPlusTree, Vec<(u32, u32)>), TuliproxError> { let xtream_path = xtream_get_file_path(storage_path, cluster); + let index_path = get_file_path_for_db_index(&xtream_path); let file_lock = app_config.file_locks.read_lock(&xtream_path).await; // Move B+Tree query and iteration to spawn_blocking to avoid blocking tokio runtime let path_clone = xtream_path.clone(); match tokio::task::spawn_blocking(move || { let _guard = file_lock; - BPlusTree::::load(&path_clone) + if !path_clone.exists() { + debug!("No xtream {cluster} storage at {}, serving empty playlist", path_clone.display()); + return Ok((BPlusTree::new(), Vec::new())); + } + read_tree_and_order(&path_clone, &index_path, cluster) }) .await { - Ok(Ok(tree)) => Ok(tree), - Ok(Err(err)) if err.kind() == std::io::ErrorKind::NotFound => { - debug!("No xtream {cluster} storage at {}, serving empty playlist", xtream_path.display()); - Ok(BPlusTree::new()) - } + Ok(Ok((tree, order))) => Ok((tree, order)), Ok(Err(err)) => Err(TuliproxError::RepositoryXtream(format!( "Failed to load xtream {cluster} storage {}: {err}", xtream_path.display() @@ -101,11 +118,11 @@ pub async fn load_xtream_target_storage( TuliproxError::Config(format!("Could not find path for target {} xtream output", target.name)) })?; - let live_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Live).await?; - let vod_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Video).await?; - let series_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Series).await?; + let (live, live_order) = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Live).await?; + let (vod, vod_order) = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Video).await?; + let (series, series_order) = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Series).await?; - Ok(PlaylistXtreamStorage { live: live_storage, vod: vod_storage, series: series_storage }) + Ok(PlaylistXtreamStorage { live, live_order, vod, vod_order, series, series_order }) } pub async fn load_m3u_target_storage( diff --git a/backend/repository/src/playlist_repository/tests/storage.rs b/backend/repository/src/playlist_repository/tests/storage.rs index efe724441..939eb9e9b 100644 --- a/backend/repository/src/playlist_repository/tests/storage.rs +++ b/backend/repository/src/playlist_repository/tests/storage.rs @@ -1,4 +1,11 @@ use super::*; +use crate::{ + open_playlist_reader, write_playlist_batch_item_upsert, write_playlist_item_update, PlaylistIteratorReader, + XtreamPlaylistIterator, +}; +use futures::StreamExt; +use shared::model::SeriesListProjection; +use tuliprox_core::model::ProxyUserCredentials; #[tokio::test] async fn force_empty_output_filters_reject_m3u_and_strm_before_mutation() { @@ -100,3 +107,543 @@ fn media_server_playlist_file_path_uses_separate_prefix() { assert!(path.ends_with("media_server_Media_Server_Input.db")); } + +#[tokio::test] +async fn cache_load_drops_series_details_while_disk_keeps_them() { + let dir = tempdir().expect("tempdir"); + let app_config = app_config_in(dir.path()); + let target = target_with_options( + "series-cache-load-target", + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + true, + ); + let storage_path = { + let config = app_config.config.load(); + xtream_get_storage_path(&config, &target.name).expect("storage path") + }; + let series_path = crate::xtream_get_file_path(&storage_path, XtreamCluster::Series); + tokio::fs::create_dir_all(series_path.parent().expect("parent dir")).await.expect("create dir"); + + let details = SeriesStreamDetailProperties { year: Some(2024), seasons: None, episodes: None }; + let mut item = crate::test_config::xtream_series_item(55, Some(details)); + item.name = "Test Series".intern(); + item.title = "Test Series".intern(); + let mut tree = crate::BPlusTree::new(); + tree.insert(55u32, item.clone()); + tree.store_with_index(&series_path, |pli| pli.source_ordinal).expect("store series tree"); + + // Load storage into memory cache + let storage = load_xtream_target_storage(&app_config, &target).await.expect("load xtream storage"); + + // Cache item has details == None + let cached_item = storage.series.query(&55).expect("item in cache"); + assert!(!cached_item.has_details(), "cache item must have details == None"); + match &cached_item.additional_properties { + Some(StreamProperties::Series(props)) => assert_eq!(props.details, None), + other => panic!("expected Series properties, got {other:?}"), + } + + // Disk item still has details + let mut disk_query = BPlusTreeQuery::::try_new(&series_path).expect("open disk query"); + let disk_item = disk_query.query(&55).expect("query succeeds").expect("item found on disk"); + assert!(disk_item.has_details(), "disk item must keep details"); + match &disk_item.additional_properties { + Some(StreamProperties::Series(props)) => assert!(props.details.is_some()), + other => panic!("expected Series properties, got {other:?}"), + } +} + +#[tokio::test] +async fn xtream_get_item_for_stream_id_with_memory_cache_returns_details_for_series_info() { + let dir = tempdir().expect("tempdir"); + let app_config = Arc::new(app_config_in(dir.path())); + let target = target_with_options( + "series-lookup-target", + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + true, + ); + let (target_path, storage_path) = { + let config = app_config.config.load(); + ( + get_target_storage_path(&config, &target.name).expect("target path"), + xtream_get_storage_path(&config, &target.name).expect("storage path"), + ) + }; + let series_path = crate::xtream_get_file_path(&storage_path, XtreamCluster::Series); + let mapping_path = crate::get_target_id_mapping_file(&target_path); + tokio::fs::create_dir_all(series_path.parent().expect("parent dir")).await.expect("create dir"); + + // 1. Write series item with details to disk + let details = SeriesStreamDetailProperties { year: Some(2024), seasons: None, episodes: None }; + let mut item = crate::test_config::xtream_series_item(99, Some(details)); + item.name = "Detailed Series".intern(); + item.title = "Detailed Series".intern(); + let mut tree = crate::BPlusTree::new(); + tree.insert(99u32, item.clone()); + tree.store_with_index(&series_path, |pli| pli.source_ordinal).expect("store series tree"); + + // 2. Write id mapping to disk and populate memory cache + let mut mapping_tree = crate::BPlusTree::new(); + let record = VirtualIdRecord { + virtual_id: VirtualId::new(99), + provider_id: 99, + uuid: UUIDType::default(), + item_type: PlaylistItemType::SeriesInfo, + parent_virtual_id: VirtualId::default(), + last_updated: 0, + }; + mapping_tree.insert(VirtualId::new(99), record); + mapping_tree.store(&mapping_path).expect("store id mapping"); + + let playlists = Arc::new(PlaylistStorageState::new()); + // Load storage into memory cache (so memory cache has series item WITHOUT details) + let storage = load_xtream_target_storage(&app_config, &target).await.expect("load xtream storage"); + assert!(!storage.series.query(&99).unwrap().has_details()); + + playlists.cache_playlist(&target.name, crate::PlaylistStorage::XtreamPlaylist(Box::new(storage))).await; + playlists.cache_id_mapping(&target.name, mapping_tree).await; + + // 3. Look up item: with use_memory_cache: true, series cluster falls through to disk and returns full details + let resolved_no_cluster = crate::xtream_get_item_for_stream_id(99, &app_config, &playlists, &target, None) + .await + .expect("xtream_get_item_for_stream_id succeeds without cluster"); + assert!(resolved_no_cluster.has_details(), "series item looked up without cluster must have details"); + + let resolved_with_cluster = + crate::xtream_get_item_for_stream_id(99, &app_config, &playlists, &target, Some(XtreamCluster::Series)) + .await + .expect("xtream_get_item_for_stream_id succeeds with cluster"); + assert!(resolved_with_cluster.has_details(), "series item looked up with cluster must have details"); +} + +#[tokio::test] +async fn xtream_get_item_for_stream_id_resolves_series_info_when_id_mapping_only_in_memory() { + let dir = tempdir().expect("tempdir"); + let app_config = Arc::new(app_config_in(dir.path())); + let target = target_with_options( + "series-lookup-mem-mapping-target", + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + true, + ); + let (target_path, storage_path) = { + let config = app_config.config.load(); + ( + get_target_storage_path(&config, &target.name).expect("target path"), + xtream_get_storage_path(&config, &target.name).expect("storage path"), + ) + }; + let series_path = crate::xtream_get_file_path(&storage_path, XtreamCluster::Series); + let mapping_path = crate::get_target_id_mapping_file(&target_path); + tokio::fs::create_dir_all(series_path.parent().expect("parent dir")).await.expect("create dir"); + + // 1. Write series item with details to disk + let details = SeriesStreamDetailProperties { year: Some(2024), seasons: None, episodes: None }; + let mut item = crate::test_config::xtream_series_item(99, Some(details)); + item.name = "Detailed Series".intern(); + item.title = "Detailed Series".intern(); + let mut tree = crate::BPlusTree::new(); + tree.insert(99u32, item.clone()); + tree.store_with_index(&series_path, |pli| pli.source_ordinal).expect("store series tree"); + + // 2. Populate memory cache with storage and in-memory id mapping, but DO NOT store id mapping to disk + let mut mapping_tree = crate::BPlusTree::new(); + let record = VirtualIdRecord { + virtual_id: VirtualId::new(99), + provider_id: 99, + uuid: UUIDType::default(), + item_type: PlaylistItemType::SeriesInfo, + parent_virtual_id: VirtualId::default(), + last_updated: 0, + }; + mapping_tree.insert(VirtualId::new(99), record); + + let playlists = Arc::new(PlaylistStorageState::new()); + let storage = load_xtream_target_storage(&app_config, &target).await.expect("load xtream storage"); + assert!(!storage.series.query(&99).unwrap().has_details()); + + playlists.cache_playlist(&target.name, crate::PlaylistStorage::XtreamPlaylist(Box::new(storage))).await; + playlists.cache_id_mapping(&target.name, mapping_tree).await; + + // Verify on-disk id mapping file is absent + assert!(!mapping_path.exists(), "on-disk id mapping file must not exist"); + + // 3. Series info lookup without cluster must resolve via in-memory mapping and return item from disk + let resolved = crate::xtream_get_item_for_stream_id(99, &app_config, &playlists, &target, None) + .await + .expect("series info lookup succeeds when id mapping exists only in memory"); + assert_eq!(resolved.virtual_id.get(), 99); + assert!(resolved.has_details(), "series item looked up must have details"); +} + +#[tokio::test] +async fn xtream_get_item_for_stream_id_disk_resolves_catchup_with_live_cluster() { + let dir = tempdir().expect("tempdir"); + let app_config = Arc::new(app_config_in(dir.path())); + let target = target_with_options( + "catchup-disk-target", + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + false, + ); + let (target_path, storage_path) = { + let config = app_config.config.load(); + ( + get_target_storage_path(&config, &target.name).expect("target path"), + xtream_get_storage_path(&config, &target.name).expect("storage path"), + ) + }; + let live_path = crate::xtream_get_file_path(&storage_path, XtreamCluster::Live); + let mapping_path = crate::get_target_id_mapping_file(&target_path); + tokio::fs::create_dir_all(live_path.parent().expect("parent dir")).await.expect("create dir"); + + let parent_virtual_id = 100u32; + let catchup_virtual_id = 200u32; + let mapping_provider_id = 555u32; + + let mut parent_item = crate::test_config::xtream_playlist_item(parent_virtual_id); + parent_item.provider_id = 111; + parent_item.item_type = PlaylistItemType::Live; + let mut tree = crate::BPlusTree::new(); + tree.insert(parent_virtual_id, parent_item); + tree.store(&live_path).expect("store live tree"); + + let mut mapping_tree = crate::BPlusTree::new(); + let record = VirtualIdRecord { + virtual_id: VirtualId::new(catchup_virtual_id), + provider_id: mapping_provider_id, + uuid: UUIDType::default(), + item_type: PlaylistItemType::Catchup, + parent_virtual_id: VirtualId::new(parent_virtual_id), + last_updated: 0, + }; + mapping_tree.insert(VirtualId::new(catchup_virtual_id), record); + mapping_tree.store(&mapping_path).expect("store id mapping"); + + let playlists = Arc::new(PlaylistStorageState::new()); + + let item = crate::xtream_get_item_for_stream_id( + catchup_virtual_id, + &app_config, + &playlists, + &target, + Some(XtreamCluster::Live), + ) + .await + .expect("catchup item lookup succeeds on disk path"); + + assert_eq!(item.item_type, PlaylistItemType::Catchup); + assert_eq!(item.provider_id, mapping_provider_id); + assert_eq!(item.virtual_id, VirtualId::new(catchup_virtual_id)); +} + +#[tokio::test] +async fn xtream_get_item_for_stream_id_disk_resolves_local_series_episode_via_parent() { + let dir = tempdir().expect("tempdir"); + let app_config = Arc::new(app_config_in(dir.path())); + let target = target_with_options( + "local-series-disk-target", + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + false, + ); + let (target_path, storage_path) = { + let config = app_config.config.load(); + ( + get_target_storage_path(&config, &target.name).expect("target path"), + xtream_get_storage_path(&config, &target.name).expect("storage path"), + ) + }; + let series_path = crate::xtream_get_file_path(&storage_path, XtreamCluster::Series); + let mapping_path = crate::get_target_id_mapping_file(&target_path); + tokio::fs::create_dir_all(series_path.parent().expect("parent dir")).await.expect("create dir"); + + let parent_virtual_id = 100u32; + let episode_virtual_id = 200u32; + let mapping_provider_id = 555u32; + + let mut parent_item = crate::test_config::xtream_playlist_item(parent_virtual_id); + parent_item.provider_id = 111; + parent_item.item_type = PlaylistItemType::LocalSeriesInfo; + let mut tree = crate::BPlusTree::new(); + tree.insert(parent_virtual_id, parent_item); + tree.store(&series_path).expect("store series tree"); + + let mut mapping_tree = crate::BPlusTree::new(); + let record = VirtualIdRecord { + virtual_id: VirtualId::new(episode_virtual_id), + provider_id: mapping_provider_id, + uuid: UUIDType::default(), + item_type: PlaylistItemType::LocalSeries, + parent_virtual_id: VirtualId::new(parent_virtual_id), + last_updated: 0, + }; + mapping_tree.insert(VirtualId::new(episode_virtual_id), record); + mapping_tree.store(&mapping_path).expect("store id mapping"); + + let playlists = Arc::new(PlaylistStorageState::new()); + + let item = crate::xtream_get_item_for_stream_id( + episode_virtual_id, + &app_config, + &playlists, + &target, + Some(XtreamCluster::Series), + ) + .await + .expect("local series episode item lookup succeeds on disk path"); + + assert_eq!(item.item_type, PlaylistItemType::LocalSeries); + assert_eq!(item.provider_id, mapping_provider_id); + assert_eq!(item.virtual_id, VirtualId::new(episode_virtual_id)); +} + +fn seed_test_cluster(storage_path: &Path, cluster: XtreamCluster) { + let path = crate::xtream_get_file_path(storage_path, cluster); + let mut tree = crate::BPlusTree::new(); + for (id, ordinal) in [(10, 300), (20, 100), (30, 200)] { + let mut item = match cluster { + XtreamCluster::Live => crate::test_config::xtream_playlist_item(id), + XtreamCluster::Video => { + let mut it = crate::test_config::xtream_playlist_item(id); + it.xtream_cluster = XtreamCluster::Video; + it.item_type = PlaylistItemType::Video; + it + } + XtreamCluster::Series => crate::test_config::xtream_series_item(id, None), + }; + item.source_ordinal = ordinal; + tree.insert(id, item); + } + tree.store_with_index(&path, |pli| pli.source_ordinal).expect("store with index"); +} + +async fn apply_test_upserts( + app_config: &Arc, + target: &ConfigTarget, + playlists: &PlaylistStorageState, + cluster: XtreamCluster, +) { + let mut repl = match cluster { + XtreamCluster::Live => crate::test_config::xtream_playlist_item(20), + XtreamCluster::Video => { + let mut it = crate::test_config::xtream_playlist_item(20); + it.xtream_cluster = XtreamCluster::Video; + it.item_type = PlaylistItemType::Video; + it + } + XtreamCluster::Series => crate::test_config::xtream_series_item(20, None), + }; + repl.source_ordinal = 100; + repl.title = "Updated Title".intern(); + + write_playlist_item_update(app_config, &target.name, &repl).await.expect("disk replacement"); + playlists.update_playlist_items(target, &[&repl]).await; + + let mut new1 = match cluster { + XtreamCluster::Live => crate::test_config::xtream_playlist_item(40), + XtreamCluster::Video => { + let mut it = crate::test_config::xtream_playlist_item(40); + it.xtream_cluster = XtreamCluster::Video; + it.item_type = PlaylistItemType::Video; + it + } + XtreamCluster::Series => crate::test_config::xtream_series_item(40, None), + }; + new1.source_ordinal = 50; + + let mut new2 = match cluster { + XtreamCluster::Live => crate::test_config::xtream_playlist_item(5), + XtreamCluster::Video => { + let mut it = crate::test_config::xtream_playlist_item(5); + it.xtream_cluster = XtreamCluster::Video; + it.item_type = PlaylistItemType::Video; + it + } + XtreamCluster::Series => crate::test_config::xtream_series_item(5, None), + }; + new2.source_ordinal = 250; + + write_playlist_batch_item_upsert(app_config, &target.name, cluster, &[new1.clone(), new2.clone()]) + .await + .expect("disk batch upsert"); + playlists.update_playlist_items(target, &[&new1, &new2]).await; + + if cluster == XtreamCluster::Series { + let mut episode = crate::test_config::xtream_playlist_item(500); + episode.xtream_cluster = XtreamCluster::Series; + episode.item_type = PlaylistItemType::Series; + episode.source_ordinal = 10; + write_playlist_batch_item_upsert(app_config, &target.name, cluster, &[episode.clone()]) + .await + .expect("disk episode upsert"); + playlists.update_playlist_items(target, &[&episode]).await; + } +} + +async fn assert_cluster_iterators_match( + app_config: &Arc, + playlists: &PlaylistStorageState, + target: &ConfigTarget, + target_disk: &ConfigTarget, + storage_path: &Path, + cluster: XtreamCluster, + expected_order: &[u32], +) { + let path = crate::xtream_get_file_path(storage_path, cluster); + let index_path = crate::get_file_path_for_db_index(&path); + let user = ProxyUserCredentials::default(); + + // Verify the disk path uses the sorted index (not the unsorted fallback) + if cluster == XtreamCluster::Series { + let reader = open_playlist_reader::(&path, &index_path, None) + .expect("open series reader"); + assert!(matches!(reader, PlaylistIteratorReader::Sorted { .. }), "disk path fell back to unsorted for Series"); + } else { + let reader = open_playlist_reader::(&path, &index_path, None) + .expect("open cluster reader"); + assert!( + matches!(reader, PlaylistIteratorReader::Sorted { .. }), + "disk path fell back to unsorted for {cluster}" + ); + } + + // Memory iterator + let mem_iter = XtreamPlaylistIterator::new(cluster, app_config, playlists, target, None, &user) + .await + .expect("memory iterator opens"); + let mem_items: Vec = + mem_iter.collect::>().await.into_iter().map(|res| res.expect("item").0).collect(); + + // Disk iterator + let disk_iter = XtreamPlaylistIterator::new(cluster, app_config, playlists, target_disk, None, &user) + .await + .expect("disk iterator opens"); + let disk_items: Vec = + disk_iter.collect::>().await.into_iter().map(|res| res.expect("item").0).collect(); + + let mem_ids: Vec = mem_items.iter().map(|item| item.virtual_id.get()).collect(); + let disk_ids: Vec = disk_items.iter().map(|item| item.virtual_id.get()).collect(); + + assert_eq!(mem_ids, expected_order, "memory iterator output matches canonical sort order for cluster {cluster}"); + assert_eq!(disk_ids, expected_order, "disk iterator output matches canonical sort order for cluster {cluster}"); + assert_eq!(disk_ids, mem_ids, "disk iterator equals memory iterator for cluster {cluster}"); + assert_eq!(disk_items.len(), mem_items.len()); + for (disk_item, mem_item) in disk_items.iter().zip(mem_items.iter()) { + assert_eq!(disk_item.virtual_id, mem_item.virtual_id); + assert_eq!(disk_item.name, mem_item.name); + assert_eq!(disk_item.title, mem_item.title); + assert_eq!(disk_item.source_ordinal, mem_item.source_ordinal); + assert_eq!(disk_item.item_type, mem_item.item_type); + } +} + +async fn assert_raw_series_contains_all_keys(app_config: &AppConfig, target: &ConfigTarget) { + let stream = crate::iter_raw_xtream_target_playlist(app_config, target, XtreamCluster::Series) + .await + .expect("raw series stream opens"); + let raw_items: Vec = + stream.collect::>().await.into_iter().map(|res| res.expect("item")).collect(); + let raw_ids: Vec = raw_items.iter().map(|item| item.virtual_id.get()).collect(); + + // Raw target iterator iterates the tree through the sorted index; assert every key + // in the tree (including episode 500) is yielded in (source_ordinal, virtual_id) order. + assert_eq!(raw_ids, vec![500, 40, 20, 30, 5, 10]); +} + +async fn assert_repositioning_moves_item( + app_config: &Arc, + playlists: &PlaylistStorageState, + target: &ConfigTarget, + target_disk: &ConfigTarget, + storage_path: &Path, +) { + let mut moved = crate::test_config::xtream_series_item(30, None); + moved.source_ordinal = 20; + moved.title = "Moved Series Title".intern(); + + write_playlist_item_update(app_config, &target.name, &moved).await.expect("disk reposition update"); + playlists.update_playlist_items(target, &[&moved]).await; + + // After moving key 30 from source_ordinal 200 to 20, it precedes key 40 (source_ordinal 50) + let new_expected_order = [30, 40, 20, 5, 10]; + assert_cluster_iterators_match( + app_config, + playlists, + target, + target_disk, + storage_path, + XtreamCluster::Series, + &new_expected_order, + ) + .await; +} + +#[tokio::test] +async fn disk_and_memory_list_output_match_canonical_sort_order_after_upserts() { + let dir = tempdir().expect("tempdir"); + let app_config = Arc::new(app_config_in(dir.path())); + let target = target_with_options( + "sort-order-target", + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + true, + ); + let storage_path = { + let config = app_config.config.load(); + xtream_get_storage_path(&config, &target.name).expect("storage path") + }; + tokio::fs::create_dir_all(&storage_path).await.expect("create storage dir"); + + for cluster in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { + seed_test_cluster(&storage_path, cluster); + } + + let storage = load_xtream_target_storage(&app_config, &target).await.expect("load xtream storage"); + let playlists = Arc::new(PlaylistStorageState::new()); + playlists.cache_playlist(&target.name, crate::PlaylistStorage::XtreamPlaylist(Box::new(storage))).await; + + for cluster in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { + apply_test_upserts(&app_config, &target, &playlists, cluster).await; + } + + let expected_order = [40, 20, 30, 5, 10]; + let mut target_disk = target.clone(); + target_disk.use_memory_cache = false; + + for cluster in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { + assert_cluster_iterators_match( + &app_config, + &playlists, + &target, + &target_disk, + &storage_path, + cluster, + &expected_order, + ) + .await; + } + + assert_raw_series_contains_all_keys(&app_config, &target).await; + assert_repositioning_moves_item(&app_config, &playlists, &target, &target_disk, &storage_path).await; +} diff --git a/backend/repository/src/playlist_stream.rs b/backend/repository/src/playlist_stream.rs index 1ab953d3a..04f1364b3 100644 --- a/backend/repository/src/playlist_stream.rs +++ b/backend/repository/src/playlist_stream.rs @@ -2,7 +2,10 @@ use super::{BPlusTreeDiskIteratorOwned, BPlusTreeQuery, SortedIndexIterator}; use futures::Stream; use log::error; use serde::{Deserialize, Serialize}; -use shared::error::TuliproxError; +use shared::{ + error::TuliproxError, + model::{SeriesListProjection, XtreamCluster, XtreamPlaylistItem}, +}; use std::{ io, path::Path, @@ -171,6 +174,53 @@ where Ok(PlaylistIteratorReader::Unsorted(query.disk_iter())) } +fn read_xtream_cluster_entries( + xtream_path: &Path, + index_path: &Path, + cluster: XtreamCluster, + mut visitor: F, +) -> Result<(), TuliproxError> +where + V: Serialize + for<'de> Deserialize<'de> + Clone + Into, + F: FnMut(u32, XtreamPlaylistItem) -> bool, +{ + let reader = open_playlist_reader::( + xtream_path, + index_path, + Some("Sorted index error, falling back to unsorted"), + )?; + for entry in reader { + match entry { + Ok((key, item)) => { + if !visitor(key, item.into()) { + break; + } + } + Err(err) => { + error!("Skipping unreadable Xtream playlist entry in cluster {cluster}: {err}"); + } + } + } + Ok(()) +} + +/// Helper that opens an Xtream cluster reader (`SeriesListProjection` for Series) and visits entries. +pub fn visit_xtream_cluster_entries( + xtream_path: &Path, + index_path: &Path, + cluster: XtreamCluster, + visitor: F, +) -> Result<(), TuliproxError> +where + F: FnMut(u32, XtreamPlaylistItem) -> bool, +{ + if cluster == XtreamCluster::Series { + read_xtream_cluster_entries::(xtream_path, index_path, cluster, visitor) + } else { + read_xtream_cluster_entries::(xtream_path, index_path, cluster, visitor) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/repository/src/test_config.rs b/backend/repository/src/test_config.rs index d35067dd5..afb5442a2 100644 --- a/backend/repository/src/test_config.rs +++ b/backend/repository/src/test_config.rs @@ -103,3 +103,18 @@ pub(crate) fn xtream_playlist_item(id: u32) -> XtreamPlaylistItem { upstream_user_agent: None, } } + +/// Series Xtream playlist item whose provider and virtual id are `id`. +pub(crate) fn xtream_series_item( + id: u32, + details: Option, +) -> XtreamPlaylistItem { + let mut item = xtream_playlist_item(id); + item.xtream_cluster = XtreamCluster::Series; + item.item_type = PlaylistItemType::SeriesInfo; + item.url = "".intern(); + let series_props = + shared::model::SeriesStreamProperties { name: format!("Series {id}").intern(), details, ..Default::default() }; + item.additional_properties = Some(shared::model::StreamProperties::Series(Box::new(series_props))); + item +} diff --git a/backend/repository/src/xtream_playlist_iterator.rs b/backend/repository/src/xtream_playlist_iterator.rs index 75955ec20..5a3184066 100644 --- a/backend/repository/src/xtream_playlist_iterator.rs +++ b/backend/repository/src/xtream_playlist_iterator.rs @@ -1,12 +1,12 @@ use crate::{ - get_file_path_for_db_index, open_playlist_reader, user_get_bouquet_filter, xtream_get_file_path, - xtream_get_item_for_stream_id, xtream_get_storage_path, LockedReceiverStream, PlaylistStorageState, + get_file_path_for_db_index, user_get_bouquet_filter, xtream_get_file_path, xtream_get_item_for_stream_id, + xtream_get_storage_path, LockedReceiverStream, PlaylistStorageState, }; use futures::{Stream, StreamExt}; use log::error; use shared::{ error::TuliproxError, - model::{PlaylistItemType, TargetType, XtreamCluster, XtreamPlaylistItem}, + model::{TargetType, XtreamCluster, XtreamPlaylistItem}, }; use std::{ collections::HashSet, @@ -20,8 +20,13 @@ use tuliprox_core::{ utils::prepare_xtream_resource_hosts, }; +enum XtreamPlaylistIteratorInner { + Disk(LockedReceiverStream>), + Memory(std::vec::IntoIter), +} + pub struct XtreamPlaylistIterator { - inner: LockedReceiverStream>, + inner: XtreamPlaylistIteratorInner, } fn is_cluster_allowed_for_user(user: &ProxyUserCredentials, cluster: XtreamCluster) -> bool { @@ -64,10 +69,7 @@ impl XtreamItemFilter { #[inline] fn matches(&self, item: &XtreamPlaylistItem) -> bool { - // We can't serve episodes within series - if self.cluster == XtreamCluster::Series - && !matches!(item.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) - { + if !crate::playlist_mem_cache::is_cluster_order_eligible(self.cluster, item.item_type) { return false; } @@ -91,28 +93,79 @@ impl XtreamItemFilter { } } -impl XtreamPlaylistIterator { - fn empty() -> Self { - // Ghost Channel pattern - // When you immediately drop the sender with `_`, the channel is closed and receiver gets None. - let (_tx, rx) = mpsc::channel::>(1); - Self { inner: LockedReceiverStream::new_empty(rx) } +fn stream_playlist_entries( + xtream_path: &std::path::Path, + index_path: &std::path::Path, + cluster: XtreamCluster, + item_filter: &XtreamItemFilter, + tx: &mpsc::Sender>, +) { + let mut pending: Option = None; + let visit_result = crate::visit_xtream_cluster_entries(xtream_path, index_path, cluster, |_key, item| { + if !item_filter.matches(&item) { + return true; + } + + if let Some(prev) = pending.replace(item) { + if tx.blocking_send(Ok((prev, true))).is_err() { + return false; + } + } + true + }); + + if let Err(err) = visit_result { + error!("Failed to open Xtream playlist DB {} (cluster {cluster}): {err}", xtream_path.display()); + let _ = tx.blocking_send(Err(err)); + return; } + if let Some(last) = pending { + let _ = tx.blocking_send(Ok((last, false))); + } +} + +impl XtreamPlaylistIterator { + fn empty() -> Self { Self { inner: XtreamPlaylistIteratorInner::Memory(Vec::new().into_iter()) } } + pub async fn new( cluster: XtreamCluster, app_config: &AppConfig, + playlists: &PlaylistStorageState, target: &ConfigTarget, category_id: Option, user: &ProxyUserCredentials, ) -> Result { - // TODO use playlist memory cache and keep sorted if !is_cluster_allowed_for_user(user, cluster) { return Ok(Self::empty()); } debug_assert!(target.get_xtream_output().is_some()); let config = app_config.config.load(); + let item_filter = XtreamItemFilter::new(&config, cluster, category_id, user).await; + + if target.use_memory_cache { + let memory_items = { + let data = playlists.data.read().await; + data.get(&target.name).and_then(|storage| storage.xtream.as_ref()).map(|xtream_storage| { + let (tree, order) = xtream_storage.get_cluster(cluster); + let mut matching = Vec::new(); + for (_, key) in order { + if let Some(item) = tree.query(key) { + if item_filter.matches(item) { + matching.push(item.clone()); + } + } + } + matching + }) + }; + + if let Some(items) = memory_items { + return Ok(Self { inner: XtreamPlaylistIteratorInner::Memory(items.into_iter()) }); + } + } + if let Some(storage_path) = xtream_get_storage_path(&config, target.name.as_str()) { let xtream_path = xtream_get_file_path(&storage_path, cluster); if !xtream_path.exists() { @@ -123,8 +176,6 @@ impl XtreamPlaylistIterator { let file_lock = Arc::new(app_config.file_locks.read_lock(&xtream_path).await); let bg_lock = Arc::clone(&file_lock); - let item_filter = XtreamItemFilter::new(&config, cluster, category_id, user).await; - let xtream_path = xtream_path.clone(); let index_path = get_file_path_for_db_index(&xtream_path); let (tx, rx) = mpsc::channel::>(256); @@ -133,46 +184,7 @@ impl XtreamPlaylistIterator { let join_error_tx = tx.clone(); let handle = task::spawn_blocking(move || { let _guard = bg_lock; - let reader = match open_playlist_reader::( - &xtream_path, - &index_path, - Some("Sorted index error, falling back to unsorted"), - ) { - Ok(reader) => reader, - Err(err) => { - error!( - "Failed to open Xtream playlist DB {} (cluster {cluster}): {err}", - xtream_path.display() - ); - let _ = tx.blocking_send(Err(err)); - return; - } - }; - - let mut pending: Option = None; - for entry in reader { - let item = match entry { - Ok((_, item)) => item, - Err(err) => { - error!("Skipping unreadable Xtream playlist entry: {err}"); - continue; - } - }; - - if !item_filter.matches(&item) { - continue; - } - - if let Some(prev) = pending.replace(item) { - if tx.blocking_send(Ok((prev, true))).is_err() { - return; - } - } - } - - if let Some(last) = pending { - let _ = tx.blocking_send(Ok((last, false))); - } + stream_playlist_entries(&xtream_path, &index_path, cluster, &item_filter, &tx); }); tokio::spawn(async move { if let Err(err) = handle.await { @@ -189,7 +201,7 @@ impl XtreamPlaylistIterator { } }); - Ok(Self { inner: LockedReceiverStream::new_shared(rx, file_lock) }) + Ok(Self { inner: XtreamPlaylistIteratorInner::Disk(LockedReceiverStream::new_shared(rx, file_lock)) }) } else { Err(TuliproxError::Config(format!("Failed to find xtream storage for target {}", target.name))) } @@ -247,7 +259,16 @@ impl<'a> XtreamUserItemLookup<'a> { impl Stream for XtreamPlaylistIterator { type Item = Result<(XtreamPlaylistItem, bool), TuliproxError>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_next(cx) + match &mut self.inner { + XtreamPlaylistIteratorInner::Disk(stream) => Pin::new(stream).poll_next(cx), + XtreamPlaylistIteratorInner::Memory(iter) => { + let next_item = iter.next().map(|item| { + let has_next = !iter.as_slice().is_empty(); + (item, has_next) + }); + Poll::Ready(next_item.map(Ok)) + } + } } } @@ -261,6 +282,7 @@ impl XtreamPlaylistJsonIterator { pub async fn new( cluster: XtreamCluster, app_config: &Arc, + playlists: &PlaylistStorageState, target: &ConfigTarget, category_id: Option, user: &ProxyUserCredentials, @@ -279,7 +301,7 @@ impl XtreamPlaylistJsonIterator { user, encrypt_secret, )?); - let raw = XtreamPlaylistIterator::new(cluster, app_config, target, category_id, user).await?; + let raw = XtreamPlaylistIterator::new(cluster, app_config, playlists, target, category_id, user).await?; let inner = raw.then(move |entry| { let options = Arc::clone(&options); async move { @@ -303,11 +325,11 @@ impl Stream for XtreamPlaylistJsonIterator { #[cfg(test)] mod tests { - use super::{is_cluster_allowed_for_user, XtreamItemFilter, XtreamPlaylistIterator}; + use super::{is_cluster_allowed_for_user, XtreamItemFilter, XtreamPlaylistIterator, XtreamPlaylistIteratorInner}; use crate::{ bplustree::BPlusTree, test_config::{app_config_in, target_with_output, xtream_playlist_item}, - xtream_get_file_path, xtream_get_storage_path, LockedReceiverStream, + xtream_get_file_path, xtream_get_storage_path, LockedReceiverStream, PlaylistStorageState, }; use futures::{poll, StreamExt}; use shared::{ @@ -402,7 +424,8 @@ mod tests { assert!(tx.send(Err(shared::error::TuliproxError::RepositoryXtream("corrupt page"))).await.is_ok()); drop(tx); - let mut iterator = XtreamPlaylistIterator { inner: LockedReceiverStream::new_empty(rx) }; + let mut iterator = + XtreamPlaylistIterator { inner: XtreamPlaylistIteratorInner::Disk(LockedReceiverStream::new_empty(rx)) }; assert!(iterator.next().await.is_some_and(|entry| entry.is_ok())); assert!(iterator.next().await.is_some_and(|entry| entry.is_err())); assert!(iterator.next().await.is_none()); @@ -431,6 +454,7 @@ mod tests { let mut iterator = XtreamPlaylistIterator::new( XtreamCluster::Live, &app_config, + &PlaylistStorageState::default(), &target, None, &ProxyUserCredentials::default(), @@ -449,4 +473,214 @@ mod tests { drop(guard); assert!(app_config.file_locks.try_write_lock(&xtream_path).await.is_ok(), "no read guard may be left behind"); } + + fn sample_series_item_with_details(id: u32) -> XtreamPlaylistItem { + use shared::model::{ + SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamDetailSeasonProperties, + SeriesStreamProperties, StreamProperties, + }; + + let details = SeriesStreamDetailProperties { + year: Some(2024), + seasons: Some(vec![SeriesStreamDetailSeasonProperties { + name: "Season 1".intern(), + season_number: 1, + episode_count: 5, + overview: Some("Overview".intern()), + air_date: Some("2024-01-01".intern()), + cover: Some("http://example.test/s1.jpg".intern()), + cover_tmdb: None, + cover_big: None, + duration: None, + }]), + episodes: Some(vec![SeriesStreamDetailEpisodeProperties { + id: 501, + episode_num: 1, + season: 1, + title: "Episode 1".intern(), + container_extension: "mp4".intern(), + custom_sid: None, + added: "1700000000".intern(), + direct_source: "".intern(), + tmdb: Some(999), + release_date: "2024-01-01".intern(), + series_release_date: None, + plot: Some("Plot".intern()), + crew: None, + duration_secs: 3600, + duration: "60".intern(), + movie_image: "http://example.test/e1.jpg".intern(), + bitrate: 4500, + rating: Some(8.0), + video: None, + audio: None, + }]), + }; + let series_props = SeriesStreamProperties { + name: "Test Series".intern(), + category_id: 1, + series_id: id, + cast: "Star".intern(), + genre: Some("Sci-Fi".intern()), + details: Some(details), + ..Default::default() + }; + XtreamPlaylistItem { + virtual_id: VirtualId::new(id), + provider_id: id, + name: "Test Series".intern(), + logo: "http://example.test/logo.png".intern(), + logo_small: "".intern(), + group: "Sci-Fi".intern(), + title: "Test Series".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: format!("http://example.test/series/{id}").intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(series_props))), + item_type: PlaylistItemType::SeriesInfo, + category_id: 1, + input_name: "test-input".intern(), + channel_no: 10, + source_ordinal: 0, + input_stream_id: id.to_string().intern(), + upstream_user_agent: None, + } + } + + #[tokio::test] + async fn series_iterator_skips_episode_details() { + use shared::model::StreamProperties; + + let temp = tempfile::tempdir().expect("temp dir should be created"); + let app_config = app_config_in(temp.path()); + let target = target_with_output( + "series-target", + TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, filter: None }), + ); + let storage_path = xtream_get_storage_path(&app_config.config.load(), &target.name).expect("storage path"); + let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Series); + std::fs::create_dir_all(xtream_path.parent().expect("parent dir")).expect("storage dir should be created"); + + let mut tree = BPlusTree::new(); + tree.insert(77u32, sample_series_item_with_details(77)); + tree.store(&xtream_path).expect("series playlist should persist"); + + let mut iterator = XtreamPlaylistIterator::new( + XtreamCluster::Series, + &app_config, + &PlaylistStorageState::default(), + &target, + None, + &ProxyUserCredentials::default(), + ) + .await + .expect("iterator should open"); + + let entry = iterator.next().await.expect("entry should be yielded").expect("entry is ok"); + let (item, has_next) = entry; + assert!(!has_next); + + // Header fields intact + assert_eq!(item.virtual_id, VirtualId::new(77)); + assert_eq!(item.provider_id, 77); + assert_eq!(item.name.as_ref(), "Test Series"); + assert_eq!(item.category_id, 1); + assert_eq!(item.item_type, PlaylistItemType::SeriesInfo); + + // Details skipped + match item.additional_properties { + Some(StreamProperties::Series(series)) => { + assert_eq!(series.name.as_ref(), "Test Series"); + assert_eq!(series.series_id, 77); + assert_eq!(series.genre.as_deref(), Some("Sci-Fi")); + assert_eq!(series.details, None, "details must be None on the list iterator"); + } + other => panic!("expected Series properties, got {other:?}"), + } + + assert!(iterator.next().await.is_none()); + } + + #[tokio::test] + async fn memory_path_yields_same_items_and_order_as_disk_path_with_sorted_index() { + use crate::{load_xtream_target_storage, PlaylistStorage}; + + let temp = tempfile::tempdir().expect("temp dir should be created"); + let app_config = app_config_in(temp.path()); + let mut target = target_with_output( + "sorted-test-target", + TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, filter: None }), + ); + let storage_path = xtream_get_storage_path(&app_config.config.load(), &target.name).expect("storage path"); + let xtream_path = xtream_get_file_path(&storage_path, XtreamCluster::Live); + std::fs::create_dir_all(xtream_path.parent().expect("parent dir")).expect("storage dir should be created"); + + let mut item1 = xtream_playlist_item(1); + item1.source_ordinal = 300; + let mut item2 = xtream_playlist_item(2); + item2.source_ordinal = 100; + let mut item3 = xtream_playlist_item(3); + item3.source_ordinal = 200; + + let mut tree = BPlusTree::new(); + tree.insert(1u32, item1); + tree.insert(2u32, item2); + tree.insert(3u32, item3); + tree.store_with_index(&xtream_path, |pli| pli.source_ordinal).expect("store_with_index succeeds"); + + let storage = load_xtream_target_storage(&app_config, &target).await.expect("load storage"); + assert_eq!(storage.live_order, vec![(100, 2), (200, 3), (300, 1)]); + + let playlists = PlaylistStorageState::new(); + playlists.cache_playlist(&target.name, PlaylistStorage::XtreamPlaylist(Box::new(storage))).await; + + // 1. Read via disk path (use_memory_cache: false) + target.use_memory_cache = false; + let disk_iter = XtreamPlaylistIterator::new( + XtreamCluster::Live, + &app_config, + &playlists, + &target, + None, + &ProxyUserCredentials::default(), + ) + .await + .expect("disk iterator opens"); + let disk_items: Vec<(XtreamPlaylistItem, bool)> = + disk_iter.collect::>().await.into_iter().map(Result::unwrap).collect(); + + // 2. Read via memory path (use_memory_cache: true) + target.use_memory_cache = true; + let mem_iter = XtreamPlaylistIterator::new( + XtreamCluster::Live, + &app_config, + &playlists, + &target, + None, + &ProxyUserCredentials::default(), + ) + .await + .expect("memory iterator opens"); + let mem_items: Vec<(XtreamPlaylistItem, bool)> = + mem_iter.collect::>().await.into_iter().map(Result::unwrap).collect(); + + assert_eq!(disk_items.len(), 3); + assert_eq!(mem_items.len(), 3); + assert_eq!( + disk_items + .iter() + .map(|(item, has_next)| (item.virtual_id.get(), item.name.as_ref(), *has_next)) + .collect::>(), + mem_items + .iter() + .map(|(item, has_next)| (item.virtual_id.get(), item.name.as_ref(), *has_next)) + .collect::>() + ); + assert_eq!( + mem_items.iter().map(|(item, has_next)| (item.virtual_id.get(), *has_next)).collect::>(), + vec![(2, true), (3, true), (1, false)] + ); + } } diff --git a/backend/repository/src/xtream_repository/read.rs b/backend/repository/src/xtream_repository/read.rs index d21d6042b..6d573a582 100644 --- a/backend/repository/src/xtream_repository/read.rs +++ b/backend/repository/src/xtream_repository/read.rs @@ -78,104 +78,66 @@ async fn xtream_read_item_for_stream_id( .map_err(|err| Error::other(format!("Query task failed for {stream_id} in {cluster}: {err}")))? } -async fn xtream_read_series_item_for_stream_id( - cfg: &AppConfig, - stream_id: u32, - storage_path: &Path, -) -> Result { - let xtream_path = xtream_get_file_path(storage_path, XtreamCluster::Series); - let file_lock = cfg.file_locks.read_lock(&xtream_path).await; - let xtream_path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result { - let _guard = file_lock; - let mut query = BPlusTreeQuery::::try_new(&xtream_path_clone)?; - match query.query_zero_copy(&stream_id) { - Ok(Some(item)) => Ok(item), - Ok(None) => Err(Error::new(ErrorKind::NotFound, format!("Item {stream_id} not found in series"))), - Err(err) => Err(Error::other(format!("Query failed for {stream_id} in series: {err}"))), - } - }) - .await - .map_err(|err| Error::other(format!("Query task failed for {stream_id} in series: {err}")))? -} - async fn xtream_get_item_for_stream_id_from_memory( virtual_id: u32, playlists: &PlaylistStorageState, target: &ConfigTarget, xtream_cluster: Option, -) -> Result, Error> { - if let Some(playlist) = playlists.data.read().await.get(target.name.as_str()) { - return match (playlist.xtream.as_ref(), playlist.id_mapping.as_ref()) { - (Some(xtream_storage), Some(id_mapping)) => { - let mapping = id_mapping - .query(&VirtualId::new(virtual_id)) - .ok_or_else(|| { - string_to_io_error(format!( - "Could not find mapping for target {} and id {}", - target.name, virtual_id - )) - })? - .clone(); - let result = match mapping.item_type { - PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => Ok(xtream_storage - .series - .query(&mapping.virtual_id.get()) - .ok_or_else(|| string_to_io_error(format!("Failed to read xtream item for id {virtual_id}")))? - .clone()), - PlaylistItemType::Series | PlaylistItemType::LocalSeries => { - log::debug!("In-memory series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", virtual_id, mapping.parent_virtual_id, mapping.provider_id); +) -> (Option, Option) { + let storage = playlists.data.read().await; + let Some(playlist) = storage.get(target.name.as_str()) else { + return (None, None); + }; + let Some(mapping) = + playlist.id_mapping.as_ref().and_then(|id_mapping| id_mapping.query(&VirtualId::new(virtual_id)).cloned()) + else { + return (None, None); + }; + let Some(xtream_storage) = playlist.xtream.as_ref() else { + return (None, Some(mapping)); + }; - if let Some(item) = xtream_storage.series.query(&virtual_id) { - Ok(item.clone()) - } else if let Some(item) = xtream_storage.series.query(&mapping.parent_virtual_id.get()) { - let mut xc_item = item.clone(); - xc_item.provider_id = mapping.provider_id; - xc_item.item_type = PlaylistItemType::Series; - xc_item.virtual_id = mapping.virtual_id; - Ok(xc_item) - } else { - Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}"))) - } - } - PlaylistItemType::Catchup => { - log::debug!("In-memory catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", virtual_id, mapping.parent_virtual_id, mapping.provider_id); - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - let item = match cluster { - XtreamCluster::Live => xtream_storage.live.query(&mapping.parent_virtual_id.get()), - XtreamCluster::Video => xtream_storage.vod.query(&mapping.parent_virtual_id.get()), - XtreamCluster::Series => xtream_storage.series.query(&mapping.parent_virtual_id.get()), - }; + let result: Option = match mapping.item_type { + PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => None, + PlaylistItemType::Series | PlaylistItemType::LocalSeries => { + log::debug!( + "In-memory series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", + virtual_id, + mapping.parent_virtual_id, + mapping.provider_id + ); + xtream_storage.series.query(&virtual_id).cloned() + } + PlaylistItemType::Catchup => { + log::debug!( + "In-memory catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", + virtual_id, + mapping.parent_virtual_id, + mapping.provider_id + ); + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + let item = xtream_storage.get_cluster(cluster).0.query(&mapping.parent_virtual_id.get()); - if let Some(pl_item) = item { - let mut xc_item = pl_item.clone(); - xc_item.provider_id = mapping.provider_id; - xc_item.item_type = PlaylistItemType::Catchup; - xc_item.virtual_id = mapping.virtual_id; - Ok(xc_item) - } else { - Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}"))) - } - } - _ => { - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - Ok((match cluster { - XtreamCluster::Live => xtream_storage.live.query(&virtual_id), - XtreamCluster::Video => xtream_storage.vod.query(&virtual_id), - XtreamCluster::Series => xtream_storage.series.query(&virtual_id), - }) - .ok_or_else(|| string_to_io_error(format!("Failed to read xtream item for id {virtual_id}")))? - .clone()) - } - }; + item.map(|pl_item| { + let mut xc_item = pl_item.clone(); + xc_item.provider_id = mapping.provider_id; + xc_item.item_type = PlaylistItemType::Catchup; + xc_item.virtual_id = mapping.virtual_id; + xc_item + }) + } + _ => { + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + xtream_storage + .get_cluster(cluster) + .0 + .query(&virtual_id) + .filter(|item| cluster != XtreamCluster::Series || !item.item_type.is_series_info()) + .cloned() + } + }; - result.map(|xpli| Some((xpli, mapping))) - } - _ => Ok(None), - }; - } - //Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}. No entry found."))) - Ok(None) + (result, Some(mapping)) } pub async fn xtream_get_item_for_stream_id( @@ -185,117 +147,165 @@ pub async fn xtream_get_item_for_stream_id( target: &ConfigTarget, xtream_cluster: Option, ) -> Result { - if target.use_memory_cache { - if let Ok(Some((playlist_item, _virtual_record))) = - xtream_get_item_for_stream_id_from_memory(virtual_id, playlists, target, xtream_cluster).await - { - return Ok(playlist_item); + let mapping_record = if target.use_memory_cache { + let (playlist_item, virtual_record) = + xtream_get_item_for_stream_id_from_memory(virtual_id, playlists, target, xtream_cluster).await; + if let Some(item) = playlist_item { + return Ok(item); } - // fall through to disk lookup on cache miss - } + virtual_record + } else { + None + }; - let config = app_config.config.load(); - let target_path = get_target_storage_path(&config, target.name.as_str()) - .ok_or_else(|| string_to_io_error(format!("Could not find path for target {}", target.name)))?; - let storage_path = xtream_get_storage_path(&config, target.name.as_str()) - .ok_or_else(|| string_to_io_error(format!("Could not find path for target {} xtream output", target.name)))?; - { - let result = if let Some(cluster) = xtream_cluster { - xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await - } else { - let target_id_mapping_file = get_target_id_mapping_file(&target_path); - let target_name = target.name.clone(); - let file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; - let target_id_mapping_file_clone = target_id_mapping_file.clone(); - let mapping = tokio::task::spawn_blocking(move || -> Result { - let _guard = file_lock; - let mut target_id_mapping = - BPlusTreeQuery::::try_new(&target_id_mapping_file_clone).map_err(|err| { - string_to_io_error(format!("Could not load id mapping for target {target_name} err:{err}")) - })?; - match target_id_mapping.query_zero_copy(&virtual_id) { - Ok(Some(record)) => Ok(record), - Ok(None) => Err(string_to_io_error(format!( - "Could not find mapping for target {target_name} and id {virtual_id}" - ))), - Err(err) => Err(string_to_io_error(format!("Query failed for id {virtual_id}: {err}"))), - } - }) - .await - .map_err(|err| string_to_io_error(format!("Mapping query task failed for id {virtual_id}: {err}")))??; - match mapping.item_type { - PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => { - xtream_read_series_item_for_stream_id(app_config, virtual_id, &storage_path).await - } - PlaylistItemType::Series | PlaylistItemType::LocalSeries => { - log::debug!( - "Disk series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", - virtual_id, - mapping.parent_virtual_id, - mapping.provider_id - ); + xtream_get_item_for_stream_id_from_disk(virtual_id, app_config, target, xtream_cluster, mapping_record).await +} - if let Ok(episode) = - xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, XtreamCluster::Series) - .await - { - return Ok(episode); - } +async fn read_virtual_id_record( + app_config: &Arc, + target_path: &Path, + target_name: &str, + virtual_id: u32, +) -> Result { + let target_id_mapping_file = get_target_id_mapping_file(target_path); + let target_name = target_name.to_string(); + let file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; + let target_id_mapping_file_clone = target_id_mapping_file.clone(); + tokio::task::spawn_blocking(move || -> Result { + let _guard = file_lock; + let mut target_id_mapping = BPlusTreeQuery::::try_new(&target_id_mapping_file_clone) + .map_err(|err| { + string_to_io_error(format!("Could not load id mapping for target {target_name} err:{err}")) + })?; + match target_id_mapping.query_zero_copy(&virtual_id) { + Ok(Some(record)) => Ok(record), + Ok(None) => { + Err(string_to_io_error(format!("Could not find mapping for target {target_name} and id {virtual_id}"))) + } + Err(err) => Err(string_to_io_error(format!("Query failed for id {virtual_id}: {err}"))), + } + }) + .await + .map_err(|err| string_to_io_error(format!("Mapping query task failed for id {virtual_id}: {err}")))? +} - if let Ok(mut item) = xtream_read_series_item_for_stream_id( - app_config, - mapping.parent_virtual_id.get(), - &storage_path, - ) - .await - { - item.provider_id = mapping.provider_id; - item.item_type = PlaylistItemType::Series; - item.virtual_id = mapping.virtual_id; - return Ok(item); - } +async fn resolve_mapping_item_from_disk( + app_config: &Arc, + storage_path: &Path, + virtual_id: u32, + mapping: &VirtualIdRecord, + xtream_cluster: Option, + direct_attempted: bool, +) -> Result { + match mapping.item_type { + PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => { + if direct_attempted && xtream_cluster == Some(XtreamCluster::Series) { + return Err(Error::new( + ErrorKind::NotFound, + format!("Item {virtual_id} not found in {}", XtreamCluster::Series), + )); + } + xtream_read_item_for_stream_id(app_config, virtual_id, storage_path, XtreamCluster::Series).await + } + PlaylistItemType::Series | PlaylistItemType::LocalSeries => { + log::debug!( + "Disk series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", + virtual_id, + mapping.parent_virtual_id, + mapping.provider_id + ); - return Err(Error::other(format!("Failed to find episode item with virtual-id {virtual_id}"))); - } - PlaylistItemType::Catchup => { - log::debug!( - "Disk catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", - virtual_id, - mapping.parent_virtual_id, - mapping.provider_id - ); - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - let mut item = xtream_read_item_for_stream_id( - app_config, - mapping.parent_virtual_id.get(), - &storage_path, - cluster, - ) - .await?; - item.provider_id = mapping.provider_id; - item.item_type = PlaylistItemType::Catchup; - item.virtual_id = mapping.virtual_id; - Ok(item) - } - _ => { - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await + if !(direct_attempted && xtream_cluster == Some(XtreamCluster::Series)) { + if let Ok(episode) = + xtream_read_item_for_stream_id(app_config, virtual_id, storage_path, XtreamCluster::Series).await + { + return Ok(episode); } } - }; - result + if let Ok(mut item) = xtream_read_item_for_stream_id( + app_config, + mapping.parent_virtual_id.get(), + storage_path, + XtreamCluster::Series, + ) + .await + { + item.provider_id = mapping.provider_id; + item.item_type = mapping.item_type; + item.virtual_id = mapping.virtual_id; + return Ok(item); + } + + Err(Error::other(format!("Failed to find episode item with virtual-id {virtual_id}"))) + } + PlaylistItemType::Catchup => { + log::debug!( + "Disk catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", + virtual_id, + mapping.parent_virtual_id, + mapping.provider_id + ); + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + let mut item = + xtream_read_item_for_stream_id(app_config, mapping.parent_virtual_id.get(), storage_path, cluster) + .await?; + item.provider_id = mapping.provider_id; + item.item_type = PlaylistItemType::Catchup; + item.virtual_id = mapping.virtual_id; + Ok(item) + } + _ => { + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + if direct_attempted && xtream_cluster == Some(cluster) { + return Err(Error::new(ErrorKind::NotFound, format!("Item {virtual_id} not found in {cluster}"))); + } + xtream_read_item_for_stream_id(app_config, virtual_id, storage_path, cluster).await + } } } +async fn xtream_get_item_for_stream_id_from_disk( + virtual_id: u32, + app_config: &Arc, + target: &ConfigTarget, + xtream_cluster: Option, + mapping_record: Option, +) -> Result { + let config = app_config.config.load(); + let storage_path = xtream_get_storage_path(&config, target.name.as_str()) + .ok_or_else(|| string_to_io_error(format!("Could not find path for target {} xtream output", target.name)))?; + + let direct_attempted = if let Some(cluster) = xtream_cluster { + if let Ok(item) = xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await { + return Ok(item); + } + true + } else { + false + }; + + let mapping = if let Some(record) = mapping_record { + record + } else { + let target_path = get_target_storage_path(&config, target.name.as_str()) + .ok_or_else(|| string_to_io_error(format!("Could not find path for target {}", target.name)))?; + read_virtual_id_record(app_config, &target_path, &target.name, virtual_id).await? + }; + + resolve_mapping_item_from_disk(app_config, &storage_path, virtual_id, &mapping, xtream_cluster, direct_attempted) + .await +} + pub async fn xtream_load_rewrite_playlist( cluster: XtreamCluster, app_config: &Arc, + playlists: &PlaylistStorageState, target: &ConfigTarget, category_id: Option, user: &ProxyUserCredentials, ) -> Result { - XtreamPlaylistJsonIterator::new(cluster, app_config, target, category_id, user).await + XtreamPlaylistJsonIterator::new(cluster, app_config, playlists, target, category_id, user).await } pub async fn iter_raw_xtream_target_playlist( diff --git a/backend/repository/src/xtream_repository/target_publish.rs b/backend/repository/src/xtream_repository/target_publish.rs index 2fffa74ac..54a0d7aa2 100644 --- a/backend/repository/src/xtream_repository/target_publish.rs +++ b/backend/repository/src/xtream_repository/target_publish.rs @@ -12,7 +12,7 @@ use super::{ #[cfg(windows)] use super::{encode_windows_path, sync_published_file_parent}; use crate::{ - bplustree::{get_file_path_for_db_index, BPlusTree, BPlusTreeStagingArtifacts, BPlusTreeUpdate}, + bplustree::{get_file_path_for_db_index, BPlusTree, BPlusTreeStagingArtifacts, BPlusTreeUpdate, IndexSnapshot}, error_macros::cant_write_result, playlist_backend::PlaylistKey, storage_const, @@ -23,7 +23,7 @@ use serde_json::Value; use shared::{ error::TuliproxError, model::{ClusterFlags, PlaylistGroup, XtreamCluster, XtreamPlaylistItem}, - utils::{get_u32_from_serde_value, Internable}, + utils::{get_u32_from_serde_value, sanitize_sensitive_info, Internable}, }; #[cfg(unix)] use std::fs; @@ -485,39 +485,7 @@ pub async fn write_playlist_item_update( target_name: &str, pli: &XtreamPlaylistItem, ) -> Result<(), TuliproxError> { - let storage_path = { - let config = app_config.config.load(); - ensure_xtream_storage_path(&config, target_name).await? - }; - let xtream_path = xtream_get_file_path(&storage_path, pli.xtream_cluster); - - if !file_exists_async(&xtream_path).await { - return Err(TuliproxError::RepositoryXtream(format!( - "BPlusTree file not found for update {}", - xtream_path.display() - ))); - } - - // Prepare encoded payload before opening the writer lock. - let prepared_items = - BPlusTreeUpdate::::prepare_upsert_batch(&[(&pli.virtual_id.get(), pli)]) - .map_err(|e| TuliproxError::RepositoryXtream(format!("Failed to serialize value: {e}")))?; - - // Keep FileLockManager lock for cross-operation coordination (e.g. swap + update). - let file_lock = app_config.file_locks.write_lock(&xtream_path).await; - - let xtream_path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { - let _guard = file_lock; - let mut tree = BPlusTreeUpdate::::try_new_with_backoff(&xtream_path_clone)?; - tree.upsert_batch_encoded(prepared_items)?; - Ok(()) - }) - .await - .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; - - Ok(()) + write_playlist_batch_item_upsert(app_config, target_name, pli.xtream_cluster, std::slice::from_ref(pli)).await } pub async fn write_playlist_batch_item_upsert( @@ -549,6 +517,8 @@ pub async fn write_playlist_batch_item_upsert( let prepared_items = BPlusTreeUpdate::::prepare_upsert_batch(&batch_refs) .map_err(|e| TuliproxError::RepositoryXtream(format!("Failed to serialize value: {e}")))?; + let items_info: Vec<(u32, u32)> = pli_list.iter().map(|pli| (pli.source_ordinal, pli.virtual_id.get())).collect(); + // Keep FileLockManager lock for cross-operation coordination (e.g. swap + update). let file_lock = app_config.file_locks.write_lock(&xtream_path).await; @@ -556,7 +526,30 @@ pub async fn write_playlist_batch_item_upsert( tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { let _guard = file_lock; let mut tree = BPlusTreeUpdate::::try_new_with_backoff(&xtream_path_clone)?; + let index_path = get_file_path_for_db_index(&xtream_path_clone); + let snapshot = IndexSnapshot::::read(&index_path).ok().flatten(); + // Keys that are new or moved to another `source_ordinal`; only needed when an index is kept. + let mut changed = Vec::new(); + if snapshot.is_some() { + for (source_ordinal, virtual_id) in items_info { + let moved = tree + .query(&virtual_id) + .map_err(std::io::Error::other)? + .is_none_or(|old_item| old_item.source_ordinal != source_ordinal); + if moved { + changed.push((source_ordinal, virtual_id)); + } + } + } tree.upsert_batch_encoded(prepared_items)?; + if let Some(snapshot) = snapshot { + if let Err(err) = snapshot.write_merged(&index_path, &changed) { + warn!( + "Failed to write merged index snapshot for {}: {err}", + sanitize_sensitive_info(&index_path.to_string_lossy()) + ); + } + } Ok(()) }) .await diff --git a/docs/src/configuration/reverse-proxy.md b/docs/src/configuration/reverse-proxy.md index d12845b91..4524885ce 100644 --- a/docs/src/configuration/reverse-proxy.md +++ b/docs/src/configuration/reverse-proxy.md @@ -451,8 +451,13 @@ reverse_proxy: ## 5. Resource Retries (`resource_retry`) -This block defines the retry behavior when Tuliprox proxies static resources (logos, EPG images). -It ensures transient network issues or temporary provider timeouts don't result in broken images for your clients. +This block defines the shared retry behavior for outbound requests that use the standard retry policy. +It ensures transient network issues or temporary provider timeouts do not fail a request on the first error. + +Image proxy fetches (logos, covers, EPG icons on the Web UI and player resource routes) do not use these same-URL +retries: they make one attempt per URL, fail after 15 seconds without a response, and run at most 6 concurrent +upstream fetches per host. Provider URL failover still applies. A failing image server therefore cannot hold a +client connection for minutes or receive a retry burst. ```yaml reverse_proxy: diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index 8476bac29..e53630792 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -1562,7 +1562,7 @@ sources: | `curation` | Object | No | | Optional target-wide catalog policy with Trakt and/or TMDB Trending sources. Mutually exclusive with legacy `output[].trakt`; see [target-wide curation](#target-wide-curation-curation). | | `favourites` | List | No | | Duplicates final transformed channels into dedicated favorite groups after processing is complete. This adds curated views without changing the original group structure. | | `watch` | List | No | | Watch entries for this target: channel changes in matching groups and/or matching groups that appear or disappear. Changes are reported as Messaging events; see [Watch](#329-watch-watch). | -| `use_memory_cache` | Bool | No | `false` | If enabled, the final compiled playlist is cached in RAM. This reduces disk access and improves delivery speed, especially for M3U downloads, but increases memory consumption. | +| `use_memory_cache` | Bool | No | `false` | If enabled, the final playlist is cached in RAM to speed up M3U downloads and Xtream lists (in sorted-index order). Series season and episode details remain on disk to keep RAM usage lean. | --- diff --git a/frontend/src/app/components/playlist/epg_view/mod.rs b/frontend/src/app/components/playlist/epg_view/mod.rs index df100aed0..fde8527b8 100644 --- a/frontend/src/app/components/playlist/epg_view/mod.rs +++ b/frontend/src/app/components/playlist/epg_view/mod.rs @@ -1202,7 +1202,7 @@ pub fn EpgView() -> Html { style={channel_style.clone()}>
{ if let Some(icon) = &ch.icon { - html! { {ch.title.to_string()} } + html! { {ch.title.to_string()} } } else { html!{} } }
diff --git a/shared/src/foundation/filter/eval.rs b/shared/src/foundation/filter/eval.rs index e83ba7a26..48f892b00 100644 --- a/shared/src/foundation/filter/eval.rs +++ b/shared/src/foundation/filter/eval.rs @@ -84,10 +84,7 @@ impl Leaf { let is_match = match item_type { PlaylistItemType::Live => actual.is_live(), PlaylistItemType::Video => actual.is_video(), - PlaylistItemType::Series => { - actual.is_series() - || matches!(actual, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) - } + PlaylistItemType::Series => actual.is_series() || actual.is_series_info(), _ => actual == *item_type, }; if log_enabled!(Level::Trace) { diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index a75659906..32ad58428 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -38,5 +38,6 @@ pub use item::{PlaylistEntry, PlaylistItem}; pub use kind::{PlaylistItemType, XtreamCluster}; pub use m3u::M3uPlaylistItem; pub use xtream::{ - ResourceOutputPolicy, XtreamMappingFlags, XtreamMappingFlagsSet, XtreamMappingOptions, XtreamPlaylistItem, + ResourceOutputPolicy, SeriesListProjection, XtreamMappingFlags, XtreamMappingFlagsSet, XtreamMappingOptions, + XtreamPlaylistItem, }; diff --git a/shared/src/model/playlist/kind.rs b/shared/src/model/playlist/kind.rs index 0d7746097..dccbc1d93 100644 --- a/shared/src/model/playlist/kind.rs +++ b/shared/src/model/playlist/kind.rs @@ -152,6 +152,11 @@ impl PlaylistItemType { /// True for concrete series item types (`Series` or `LocalSeries`); excludes the `SeriesInfo` containers. pub fn is_series(&self) -> bool { matches!(self, PlaylistItemType::Series | PlaylistItemType::LocalSeries) } + /// True for series info container types (`SeriesInfo` or `LocalSeriesInfo`). + pub fn is_series_info(&self) -> bool { + matches!(self, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) + } + /// Controls address tracking only. /// Do not use this to decide whether a playback request should use session-based admission /// or whether a logical playback must stay on the same provider account. diff --git a/shared/src/model/playlist/tests/identity.rs b/shared/src/model/playlist/tests/identity.rs index eaed76d58..6583d4b51 100644 --- a/shared/src/model/playlist/tests/identity.rs +++ b/shared/src/model/playlist/tests/identity.rs @@ -19,6 +19,16 @@ fn cluster_is_the_single_source_of_truth_for_every_item_type() { } } +#[test] +fn is_series_info_matches_only_series_info_types() { + use strum::IntoEnumIterator; + + for item_type in PlaylistItemType::iter() { + let expected = matches!(item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo); + assert_eq!(item_type.is_series_info(), expected, "{item_type:?}"); + } +} + #[test] fn from_stalker_uuid_is_seeded_with_input_name_not_category() { let mut item = StalkerPlaylistItem { diff --git a/shared/src/model/playlist/tests/xtream.rs b/shared/src/model/playlist/tests/xtream.rs index ddf1267eb..9c320c7a5 100644 --- a/shared/src/model/playlist/tests/xtream.rs +++ b/shared/src/model/playlist/tests/xtream.rs @@ -209,3 +209,267 @@ fn get_resource_url_does_not_trust_absolute_urls_containing_internal_thumbnail_p concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), ); } + +#[test] +fn series_list_projection_skips_details_on_rmp_decode() { + use crate::model::{ + SeriesListProjection, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, + SeriesStreamDetailSeasonProperties, SeriesStreamProperties, + }; + + let details = SeriesStreamDetailProperties { + year: Some(2024), + seasons: Some(vec![SeriesStreamDetailSeasonProperties { + name: "Season 1".intern(), + season_number: 1, + episode_count: 8, + overview: Some("Overview".intern()), + air_date: Some("2024-01-01".intern()), + cover: Some("http://example.test/s1.jpg".intern()), + cover_tmdb: None, + cover_big: None, + duration: None, + }]), + episodes: Some(vec![SeriesStreamDetailEpisodeProperties { + id: 101, + episode_num: 1, + season: 1, + title: "Pilot".intern(), + container_extension: "mkv".intern(), + custom_sid: None, + added: "1700000000".intern(), + direct_source: "".intern(), + tmdb: Some(1234), + release_date: "2024-01-01".intern(), + series_release_date: None, + plot: Some("Plot".intern()), + crew: None, + duration_secs: 3600, + duration: "60".intern(), + movie_image: "http://example.test/e1.jpg".intern(), + bitrate: 4000, + rating: Some(8.5), + video: None, + audio: None, + }]), + }; + let series_props = SeriesStreamProperties { + name: "Test Series".intern(), + category_id: 10, + series_id: 42, + cast: "Cast".intern(), + cover: "http://example.test/cover.jpg".intern(), + director: "Director".intern(), + genre: Some("Drama".intern()), + plot: Some("Series Plot".intern()), + rating: 9.0, + details: Some(details), + ..Default::default() + }; + let item = XtreamPlaylistItem { + virtual_id: VirtualId::new(42), + provider_id: 42, + name: "Test Series".intern(), + logo: "http://example.test/logo.jpg".intern(), + logo_small: "".intern(), + group: "Drama".intern(), + title: "Test Series".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://example.test/stream".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(series_props))), + item_type: PlaylistItemType::SeriesInfo, + category_id: 10, + input_name: "test-input".intern(), + channel_no: 1, + source_ordinal: 0, + input_stream_id: "42".intern(), + upstream_user_agent: None, + }; + + let encoded = rmp_serde::to_vec(&item).expect("rmp encode succeeds"); + + // Standard decode preserves details + let decoded_full: XtreamPlaylistItem = rmp_serde::from_slice(&encoded).expect("rmp decode succeeds"); + assert!(decoded_full.has_details()); + let full_series = match decoded_full.additional_properties.as_ref() { + Some(StreamProperties::Series(s)) => s, + _ => panic!("expected series properties"), + }; + assert!(full_series.details.is_some()); + + // Projection decode drops details + let projected: SeriesListProjection = rmp_serde::from_slice(&encoded).expect("projection decode succeeds"); + let decoded_projected = projected.0; + assert!(!decoded_projected.has_details()); + let projected_series = match decoded_projected.additional_properties.as_ref() { + Some(StreamProperties::Series(s)) => s, + _ => panic!("expected series properties"), + }; + assert!(projected_series.details.is_none()); + + // All other series fields are equal + let mut expected_series = full_series.clone(); + expected_series.details = None; + assert_eq!(projected_series, &expected_series); + + // Header fields are equal + assert_eq!(decoded_projected.virtual_id, decoded_full.virtual_id); + assert_eq!(decoded_projected.provider_id, decoded_full.provider_id); + assert_eq!(decoded_projected.name, decoded_full.name); + assert_eq!(decoded_projected.logo, decoded_full.logo); + assert_eq!(decoded_projected.group, decoded_full.group); + assert_eq!(decoded_projected.title, decoded_full.title); + assert_eq!(decoded_projected.url, decoded_full.url); + assert_eq!(decoded_projected.item_type, decoded_full.item_type); + assert_eq!(decoded_projected.category_id, decoded_full.category_id); + assert_eq!(decoded_projected.input_name, decoded_full.input_name); + assert_eq!(decoded_projected.channel_no, decoded_full.channel_no); + assert_eq!(decoded_projected.input_stream_id, decoded_full.input_stream_id); +} + +#[test] +fn rendered_get_series_json_contains_no_seasons_or_episodes() { + use crate::model::{ + PlaylistItemTypeSet, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, + SeriesStreamDetailSeasonProperties, SeriesStreamProperties, XtreamMappingFlags, XtreamMappingOptions, + }; + + let details = SeriesStreamDetailProperties { + year: Some(2024), + seasons: Some(vec![SeriesStreamDetailSeasonProperties { + name: "Season 1".intern(), + season_number: 1, + episode_count: 8, + overview: Some("Overview".intern()), + air_date: Some("2024-01-01".intern()), + cover: Some("http://example.test/s1.jpg".intern()), + cover_tmdb: None, + cover_big: None, + duration: None, + }]), + episodes: Some(vec![SeriesStreamDetailEpisodeProperties { + id: 101, + episode_num: 1, + season: 1, + title: "Pilot".intern(), + container_extension: "mkv".intern(), + custom_sid: None, + added: "1700000000".intern(), + direct_source: "".intern(), + tmdb: Some(1234), + release_date: "2024-01-01".intern(), + series_release_date: None, + plot: Some("Plot".intern()), + crew: None, + duration_secs: 3600, + duration: "60".intern(), + movie_image: "http://example.test/e1.jpg".intern(), + bitrate: 4000, + rating: Some(8.5), + video: None, + audio: None, + }]), + }; + let series_props = SeriesStreamProperties { + name: "Test Series".intern(), + category_id: 10, + series_id: 42, + cast: "Cast".intern(), + cover: "http://example.test/cover.jpg".intern(), + director: "Director".intern(), + genre: Some("Drama".intern()), + plot: Some("Series Plot".intern()), + rating: 9.0, + details: Some(details), + ..Default::default() + }; + let item = XtreamPlaylistItem { + virtual_id: VirtualId::new(42), + provider_id: 42, + name: "Test Series".intern(), + logo: "http://example.test/logo.jpg".intern(), + logo_small: "".intern(), + group: "Drama".intern(), + title: "Test Series".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://example.test/stream".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(series_props))), + item_type: PlaylistItemType::SeriesInfo, + category_id: 10, + input_name: "test-input".intern(), + channel_no: 1, + source_ordinal: 0, + input_stream_id: "42".intern(), + upstream_user_agent: None, + }; + + let options = XtreamMappingOptions { + flags: XtreamMappingFlags::RewriteResourceUrl.into(), + force_redirect: None, + reverse_item_types: PlaylistItemTypeSet::empty(), + resource_proxy_item_types: PlaylistItemTypeSet::empty(), + username: "user".to_string(), + password: "pass".to_string(), + base_url: "http://proxy.example".to_string(), + web_ui_request: false, + encrypt_secret: [0u8; 16], + resource_host_policies: Arc::default(), + }; + + let doc = item.to_document(&options); + let json = serde_json::to_string(&doc).expect("serialize get_series document"); + let value: serde_json::Value = serde_json::from_str(&json).expect("parse json"); + + assert!(!json.contains("\"seasons\""), "rendered get_series JSON must not contain seasons"); + assert!(!json.contains("\"episodes\""), "rendered get_series JSON must not contain episodes"); + let obj = value.as_object().expect("expected json object"); + assert!(!obj.contains_key("seasons")); + assert!(!obj.contains_key("episodes")); + assert_eq!(obj.get("name").and_then(serde_json::Value::as_str), Some("Test Series")); + assert_eq!(obj.get("series_id").and_then(serde_json::Value::as_u64), Some(42)); +} + +#[test] +fn clear_series_details_removes_details() { + use crate::model::{SeriesStreamDetailProperties, SeriesStreamProperties}; + + let details = SeriesStreamDetailProperties { year: Some(2024), seasons: None, episodes: None }; + let series_props = + SeriesStreamProperties { name: "Test Series".intern(), details: Some(details), ..Default::default() }; + let mut item = XtreamPlaylistItem { + virtual_id: VirtualId::new(1), + provider_id: 1, + name: "Test".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(series_props))), + item_type: PlaylistItemType::SeriesInfo, + category_id: 1, + input_name: "test".intern(), + channel_no: 1, + source_ordinal: 0, + input_stream_id: "1".intern(), + upstream_user_agent: None, + }; + + assert!(item.has_details()); + item.clear_series_details(); + assert!(!item.has_details()); + match item.additional_properties { + Some(StreamProperties::Series(props)) => assert_eq!(props.details, None), + _ => panic!("expected series properties"), + } +} diff --git a/shared/src/model/playlist/xtream.rs b/shared/src/model/playlist/xtream.rs index a20402507..f661a6dec 100644 --- a/shared/src/model/playlist/xtream.rs +++ b/shared/src/model/playlist/xtream.rs @@ -2,7 +2,8 @@ use super::{PlaylistEntry, PlaylistItemType, PlaylistItemTypeSet, XtreamCluster} use crate::{ create_bitset, model::{ - xtream_const, ClusterFlags, CommonPlaylistItem, StreamProperties, UUIDType, VirtualId, XtreamInfoDocument, + xtream_const, ClusterFlags, CommonPlaylistItem, SkipSeriesDetailsGuard, StreamProperties, UUIDType, VirtualId, + XtreamInfoDocument, }, utils::{ arc_str_option_serde, arc_str_serde, concat_path, generate_runtime_playlist_uuid, seal_web_ui_resource_url, @@ -258,6 +259,13 @@ impl XtreamPlaylistItem { self.additional_properties.as_ref().is_some_and(super::super::stream_properties::StreamProperties::has_details) } + #[inline] + pub fn clear_series_details(&mut self) { + if let Some(props) = self.additional_properties.as_mut() { + props.clear_series_details(); + } + } + pub fn visit_resource_urls(&self, mut visit: impl FnMut(&str)) { let mut visit_persisted = |value: &str| { if let Some(url) = super::super::persisted_resource_url(value) { @@ -404,3 +412,25 @@ impl PlaylistEntry for XtreamPlaylistItem { impl From for CommonPlaylistItem { fn from(item: XtreamPlaylistItem) -> Self { item.to_common() } } + +/// Wrapper around `XtreamPlaylistItem` that enables the `skip_series_details` +/// thread-local flag during deserialization, discarding season and episode details +/// (which are not needed on the `get_series` list path). +#[derive(Clone, Debug, Serialize)] +#[serde(transparent)] +pub struct SeriesListProjection(pub XtreamPlaylistItem); + +impl<'de> Deserialize<'de> for SeriesListProjection { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let _guard = SkipSeriesDetailsGuard::enter(); + XtreamPlaylistItem::deserialize(deserializer).map(Self) + } +} + +impl From for XtreamPlaylistItem { + #[inline] + fn from(projection: SeriesListProjection) -> Self { projection.0 } +} diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index ef3813ab1..5dfcc56f8 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -78,6 +78,12 @@ impl StreamProperties { } } + pub fn clear_series_details(&mut self) { + if let StreamProperties::Series(series) = self { + series.details = None; + } + } + pub fn prepare(&mut self) { match self { StreamProperties::Live(live) => { @@ -333,6 +339,7 @@ mod video; pub use catchup::{CatchupAttribute, CatchupProperties}; pub use live::LiveStreamProperties; pub use resources::is_resource_field_name; +pub(crate) use series::SkipSeriesDetailsGuard; use series::{first_series_episode_value, non_empty_arc, parse_season_episode_field, parse_season_field}; pub use series::{ normalize_episode_title, EpisodeStreamProperties, SeriesStreamDetailEpisodeProperties, diff --git a/shared/src/model/stream_properties/series.rs b/shared/src/model/stream_properties/series.rs index c66362876..144147343 100644 --- a/shared/src/model/stream_properties/series.rs +++ b/shared/src/model/stream_properties/series.rs @@ -9,7 +9,45 @@ use crate::{ }; use log::warn; use serde::{Deserialize, Serialize}; -use std::sync::Arc; +use std::{cell::Cell, sync::Arc}; + +thread_local! { + static SKIP_SERIES_DETAILS: Cell = const { Cell::new(false) }; +} + +/// RAII guard that sets the thread-local `SKIP_SERIES_DETAILS` flag to `true`, +/// and restores the previous value when dropped. +#[derive(Debug)] +pub(crate) struct SkipSeriesDetailsGuard { + prev: bool, +} + +impl SkipSeriesDetailsGuard { + #[must_use] + pub(crate) fn enter() -> Self { + let prev = SKIP_SERIES_DETAILS.with(|flag| flag.replace(true)); + Self { prev } + } + + #[inline] + pub(crate) fn is_enabled() -> bool { SKIP_SERIES_DETAILS.with(Cell::get) } +} + +impl Drop for SkipSeriesDetailsGuard { + fn drop(&mut self) { SKIP_SERIES_DETAILS.with(|flag| flag.set(self.prev)); } +} + +fn deserialize_series_details<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + if SkipSeriesDetailsGuard::is_enabled() { + serde::de::IgnoredAny::deserialize(deserializer)?; + Ok(None) + } else { + Option::::deserialize(deserializer) + } +} fn format_episode_code(season: u32, episode: u32) -> Option { match (season, episode) { @@ -182,7 +220,7 @@ pub struct SeriesStreamProperties { pub youtube_trailer: Arc, #[serde(default, deserialize_with = "deserialize_number_from_string")] pub tmdb: Option, - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_series_details")] pub details: Option, } @@ -474,3 +512,34 @@ pub(super) fn non_empty_arc(s: &Arc) -> Option> { Some(Arc::clone(s)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn skip_series_details_guard_restores_previous_flag() { + assert!(!SkipSeriesDetailsGuard::is_enabled()); + { + let _g1 = SkipSeriesDetailsGuard::enter(); + assert!(SkipSeriesDetailsGuard::is_enabled()); + { + let _g2 = SkipSeriesDetailsGuard::enter(); + assert!(SkipSeriesDetailsGuard::is_enabled()); + } + assert!(SkipSeriesDetailsGuard::is_enabled(), "outer guard still active"); + } + assert!(!SkipSeriesDetailsGuard::is_enabled(), "restored to false"); + } + + #[test] + fn skip_series_details_guard_unwinds_safely() { + assert!(!SkipSeriesDetailsGuard::is_enabled()); + let _ = std::panic::catch_unwind(|| { + let _guard = SkipSeriesDetailsGuard::enter(); + assert!(SkipSeriesDetailsGuard::is_enabled()); + panic!("test unwind"); + }); + assert!(!SkipSeriesDetailsGuard::is_enabled(), "restored to false after unwind"); + } +}